diff --git a/README.md b/README.md
index 1c18889a26ce23a363562327b94a335c06ca115a..b1d86fef64dff56c0cd13d7be5c2ebbceef1e432 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Sample dataset used for CoT analysis
## Prompts and Evaluation Results
-Prompts used to evaluate SummEval with GPT-4 are in prompts/summeval (by G-Eval paper)
+Prompts used to evaluate SummEval with GPT-4 & base and detailed prompts for CoT analysis are in prompts/summeval (by G-Eval paper)
Auto-CoT prompts are in prompts/cot_analysis
GPT-4 G-eval results on SummEval are in results (by G-Eval paper)
diff --git a/auto_cot.py b/auto_cot.py
index e3d437b9ec74026b6636d1ac63d3a6e8a3a823ed..944f06fa9d8e4ada3e70b5fc150336933ec0b7f1 100644
--- a/auto_cot.py
+++ b/auto_cot.py
@@ -1,10 +1,9 @@
from google import genai
from google.genai import types
from groq import Groq
+from openai import OpenAI
import os
-api_key = "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"
-
def read_prompts_from_file(filename, delimiter):
"""Reads a file and splits the content into prompts based on the delimiter."""
with open(filename, "r", encoding="utf-8") as file:
@@ -30,7 +29,7 @@ def call_gemini(prompt):
def call_Qwen(prompt):
"""Call Qwen via Groq and return the response."""
try:
- client = Groq(api_key= api_key)#os.environ.get("GROQ_API_KEY"))
+ client = Groq(api_key=os.environ.get("GROQ_API_KEY"))
response = client.chat.completions.create(
model="qwen-2.5-32b",
@@ -45,12 +44,12 @@ def call_Qwen(prompt):
return f"Groq API error: {e}"
def call_llama(prompt):
- """Call Llama via Groq and return the response."""
+ """Call Llama via Hyperbolic and return the response."""
try:
- client = Groq(api_key= api_key)#os.environ.get("GROQ_API_KEY"))
+ client = OpenAI(api_key=os.environ.get("HYPERBOLIC_API_KEY"), base_url="https://api.hyperbolic.xyz/v1")
response = client.chat.completions.create(
- model="llama3-70b-8192",
+ model="meta-llama/Meta-Llama-3-70B-Instruct",
messages=[{"role": "user", "content": prompt}],
temperature=0.0,
)
@@ -58,7 +57,7 @@ def call_llama(prompt):
return response.choices[0].message.content
except Exception as e:
- return f"Groq API error: {e}"
+ return f"Hyperbolic API error: {e}"
if __name__ == "__main__":
input_filename = 'prompts\\cot_analysis\\base_prompts.txt'
diff --git a/correlation_tables/gemini_output.txt b/correlation_tables/gemini_output.txt
index 21bd1fccf8ebecbe8573715e5401bd67e50dc32c..6c9b484ba578d2868fa903a2ce6ede2b0c3cfc04 100644
--- a/correlation_tables/gemini_output.txt
+++ b/correlation_tables/gemini_output.txt
@@ -1,91 +1,91 @@
-Filename: .\results\gemini\gemini_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.3529411764705883
+Filename: .\results\gemini\gemini_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.3529411764705883, Skipped Scores: 5
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.6866 | 0.6713 | 0.6578 |
+---------+----------+---------+
-Filename: .\results\gemini\gemini_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.5
+Filename: .\results\gemini\gemini_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.5, Skipped Scores: 1
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.9922 | 1.0 | 1.0 |
+---------+----------+---------+
-Filename: .\results\gemini\gemini_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.8596491228070176
+Filename: .\results\gemini\gemini_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.8596491228070176, Skipped Scores: 1
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.6381 | 0.6364 | 0.6267 |
+---------+----------+---------+
-Filename: .\results\gemini\llama_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.466666666666667
+Filename: .\results\gemini\llama_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.466666666666667, Skipped Scores: 24
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.5714 | 0.5714 | 0.5714 |
+---------+----------+---------+
-Filename: .\results\gemini\llama_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.5
+Filename: .\results\gemini\llama_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.5, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 1.0 | 1.0 | 1.0 |
+---------+----------+---------+
-Filename: .\results\gemini\llama_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.85
+Filename: .\results\gemini\llama_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.85, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.5857 | 0.575 | 0.5612 |
+---------+----------+---------+
-Filename: .\results\gemini\qwen_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.3333333333333335
+Filename: .\results\gemini\qwen_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.3333333333333335, Skipped Scores: 16
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.6275 | 0.6065 | 0.5855 |
+---------+----------+---------+
-Filename: .\results\gemini\qwen_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.823529411764706
+Filename: .\results\gemini\qwen_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.823529411764706, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.9768 | 0.9833 | 0.9771 |
+---------+----------+---------+
-Filename: .\results\gemini\qwen_rel_detailed.json, Dimension: relevance, Average Prediction Score: 4.022222222222222
+Filename: .\results\gemini\qwen_rel_detailed.json, Dimension: relevance, Average Prediction Score: 4.022222222222222, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.6407 | 0.6244 | 0.6148 |
+---------+----------+---------+
-Filename: .\results\gemini\scrambled_coh.json, Dimension: coherence, Average Prediction Score: 3.24
+Filename: .\results\gemini\scrambled_coh.json, Dimension: coherence, Average Prediction Score: 3.24, Skipped Scores: 16
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.7879 | 0.7855 | 0.7749 |
+---------+----------+---------+
-Filename: .\results\gemini\scrambled_con.json, Dimension: consistency, Average Prediction Score: 3.5384615384615383
+Filename: .\results\gemini\scrambled_con.json, Dimension: consistency, Average Prediction Score: 3.5384615384615383, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.969 | 0.9777 | 0.9694 |
+---------+----------+---------+
-Filename: .\results\gemini\scrambled_rel.json, Dimension: relevance, Average Prediction Score: 3.95
+Filename: .\results\gemini\scrambled_rel.json, Dimension: relevance, Average Prediction Score: 3.95, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
| 0.7872 | 0.773 | 0.7661 |
+---------+----------+---------+
-Filename: .\results\gemini\short_coh.json, Dimension: coherence, Average Prediction Score: 3.5952380952380953
+Filename: .\results\gemini\short_coh.json, Dimension: coherence, Average Prediction Score: 3.41025641025641, Skipped Scores: 9
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
-| 0.4153 | 0.4092 | 0.4065 |
+| 0.614 | 0.5931 | 0.596 |
+---------+----------+---------+
-Filename: .\results\gemini\short_con.json, Dimension: consistency, Average Prediction Score: 3.3333333333333335
+Filename: .\results\gemini\short_con.json, Dimension: consistency, Average Prediction Score: 3.6, Skipped Scores: 1
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
-| 0.9735 | 0.9809 | 0.9738 |
+| 0.9794 | 0.9851 | 0.9796 |
+---------+----------+---------+
-Filename: .\results\gemini\short_rel.json, Dimension: relevance, Average Prediction Score: 3.8333333333333335
+Filename: .\results\gemini\short_rel.json, Dimension: relevance, Average Prediction Score: 3.8292682926829267, Skipped Scores: 0
+---------+----------+---------+
| Pearson | Spearman | Kendall |
+---------+----------+---------+
-| 0.5161 | 0.5083 | 0.503 |
+| 0.5082 | 0.5004 | 0.4906 |
+---------+----------+---------+
\ No newline at end of file
diff --git a/correlation_tables/llama_output.txt b/correlation_tables/llama_output.txt
new file mode 100644
index 0000000000000000000000000000000000000000..80bb67986069a1517882f451922d783c66223f84
--- /dev/null
+++ b/correlation_tables/llama_output.txt
@@ -0,0 +1,91 @@
+
+Filename: .\results\llama\gemini_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.0454545454545454, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.6394 | 0.626 | 0.6219 |
++---------+----------+---------+
+Filename: .\results\llama\gemini_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.7142857142857144, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.7144 | 0.7222 | 0.7222 |
++---------+----------+---------+
+Filename: .\results\llama\gemini_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.675, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.7344 | 0.7322 | 0.7299 |
++---------+----------+---------+
+Filename: .\results\llama\llama_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.088888888888889, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.7767 | 0.7727 | 0.7656 |
++---------+----------+---------+
+Filename: .\results\llama\llama_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.7, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.7849 | 0.7927 | 0.7982 |
++---------+----------+---------+
+Filename: .\results\llama\llama_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.6315789473684212, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.5 | 0.4839 | 0.4853 |
++---------+----------+---------+
+Filename: .\results\llama\qwen_coh_detailed.json, Dimension: coherence, Average Prediction Score: 3.0833333333333335, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.6922 | 0.6791 | 0.6651 |
++---------+----------+---------+
+Filename: .\results\llama\qwen_con_detailed.json, Dimension: consistency, Average Prediction Score: 3.5238095238095237, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.7571 | 0.7629 | 0.7574 |
++---------+----------+---------+
+Filename: .\results\llama\qwen_rel_detailed.json, Dimension: relevance, Average Prediction Score: 3.7333333333333334, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.6286 | 0.6109 | 0.6095 |
++---------+----------+---------+
+Filename: .\results\llama\scrambled_coh.json, Dimension: coherence, Average Prediction Score: 3.1777777777777776, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.7154 | 0.705 | 0.6921 |
++---------+----------+---------+
+Filename: .\results\llama\scrambled_con.json, Dimension: consistency, Average Prediction Score: 3.5, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.8255 | 0.8333 | 0.8333 |
++---------+----------+---------+
+Filename: .\results\llama\scrambled_rel.json, Dimension: relevance, Average Prediction Score: 3.8666666666666667, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.5328 | 0.539 | 0.5402 |
++---------+----------+---------+
+Filename: .\results\llama\short_coh.json, Dimension: coherence, Average Prediction Score: 3.475, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.697 | 0.6901 | 0.6745 |
++---------+----------+---------+
+Filename: .\results\llama\short_con.json, Dimension: consistency, Average Prediction Score: 3.857142857142857, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.8255 | 0.8333 | 0.8333 |
++---------+----------+---------+
+Filename: .\results\llama\short_rel.json, Dimension: relevance, Average Prediction Score: 4.038461538461538, Skipped Scores: 0
++---------+----------+---------+
+| Pearson | Spearman | Kendall |
++---------+----------+---------+
+| 0.4965 | 0.4944 | 0.4976 |
++---------+----------+---------+
\ No newline at end of file
diff --git a/gemini_eval.py b/gemini_eval.py
index 6413697e9d4a32c0aaec485f163c03f31b76e320..92f67b952938c6687b5ece11ae71c8bead55cf57 100644
--- a/gemini_eval.py
+++ b/gemini_eval.py
@@ -22,7 +22,7 @@ sample_summeval = [summeval[i] for i in sample_indices]
ct, ignore = 0, 0
-for file in os.listdir(prompt_dir):
+for file in os.listdir(prompt_dir)[12:]:
if file.endswith(".txt"):
prompt_fp = os.path.join(prompt_dir, file)
output_fp = os.path.join(output_dir, file.replace(".txt", ".json"))
@@ -41,7 +41,7 @@ for file in os.listdir(prompt_dir):
contents=cur_prompt,
config=types.GenerateContentConfig(
temperature=2,
- max_output_tokens=5,
+ max_output_tokens=5, #Should have been slightly higher to accommodate for invalid model responses
top_p=1,
frequency_penalty=0,
presence_penalty=0,
diff --git a/llama3_eval.py b/llama3_eval.py
index 069d557b5e7e8bc868bccf9456cb7bb309ec580d..2e260dd8b89eb38018f0f66d5a2cf8bede46e5e7 100644
--- a/llama3_eval.py
+++ b/llama3_eval.py
@@ -1,16 +1,20 @@
from groq import Groq
+from openai import OpenAI
import random
import os
import json
import tqdm
import time
-client = Groq(api_key="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")
+client = OpenAI(
+ api_key=os.environ["HYPERBOLIC_API_KEY"],
+ base_url="https://api.hyperbolic.xyz/v1",
+ )
prompt_dir = 'prompts\\cot_analysis'
output_dir = 'results\\llama'
summeval_dir = 'data\\summeval.json'
-model = 'llama3-70b-8192'
+model = 'meta-llama/Meta-Llama-3-70B-Instruct'
os.makedirs(output_dir, exist_ok=True)
summeval = json.load(open(summeval_dir))
@@ -21,7 +25,7 @@ sample_summeval = [summeval[i] for i in sample_indices]
ct, ignore = 0, 0
-for file in os.listdir(prompt_dir):
+for file in os.listdir(prompt_dir)[12:]:
if file.endswith(".txt"):
prompt_fp = os.path.join(prompt_dir, file)
output_fp = os.path.join(output_dir, file.replace(".txt", ".json"))
@@ -38,8 +42,10 @@ for file in os.listdir(prompt_dir):
try:
_response = client.chat.completions.create(
model=model,
- messages=[{"role": "system", "content": "You are a helpful assistant that compares the Source Text and Summary based on the prompt given. You only give numerical values as response, based on the Evalation Steps given, and do not respond with text."},
- {"role": "user", "content": cur_prompt}],
+ messages=[
+ {"role": "system", "content": "You are a helpful assistant that compares the Source Text and Summary based on the prompt given. You only give numerical values as response, based on the Evalation Steps given, and do not respond with text."},
+ {"role": "user", "content": cur_prompt}
+ ],
temperature=0,
max_completion_tokens=10,
top_p=1,
@@ -47,7 +53,7 @@ for file in os.listdir(prompt_dir):
presence_penalty=0,
stop=None,
#logprobs=True, Not yet supported
- #n=20 Currently only n=1 supported
+ #n=20 Possible to estimate logprobs via high temperature and n
)
time.sleep(0.5)
diff --git a/meta_eval_summeval.py b/meta_eval_summeval.py
index 30fba412b77dc571ee21120a447b31bc880d94d5..ae94978a163d9ff72076b8bf82b20fb12865b21b 100644
--- a/meta_eval_summeval.py
+++ b/meta_eval_summeval.py
@@ -9,7 +9,7 @@ from glob import glob
def calculate_correlation(pred_score, human_score, result):
assert len(pred_score) == len(human_score)
- #Part of original code, not sure how to fix without breaking it
+ #Throws error: Part of original code, not sure how to fix without breaking it
if (len(result) == 0) or result is None:
result = {'pearson': 0, 'spearman': 0, 'kendalltau': 0}
result['pearson'] += pearsonr(pred_score, human_score)[0]
@@ -18,7 +18,7 @@ def calculate_correlation(pred_score, human_score, result):
return result
-#Very problematic way of doing this, but wasn't sure how else to do it (alternative would be to change the filenames to name the dimensions directly)
+#Problematic approach, but wasn't sure how else to do it (alternative is changing the filenames to name the dimensions directly)
def dimension_from_filename(filename):
#Asked ChatGPT for this Regex
@@ -37,24 +37,21 @@ def dimension_from_filename(filename):
else:
return None
-def print_correlations(result, n, input_file, dimension, avg_pred_score, output_file):
+def print_correlations(result, n, input_file, dimension, avg_pred_score, skipped_scores, output_file):
table = PrettyTable(['Pearson', 'Spearman', 'Kendall'])
if (n == 0):
n = 1
table.add_row(
[round(result['pearson'] / n, 4), round(result['spearman'] / n, 4), round(result['kendalltau'] / n, 4)])
- output_file.write(f"\nFilename: {input_file}, Dimension: {dimension}, Average Prediction Score: {avg_pred_score}\n")
+ output_file.write(f"\nFilename: {input_file}, Dimension: {dimension}, Average Prediction Score: {avg_pred_score}, Skipped Scores: {skipped_scores}\n")
output_file.write(str(table))
def parse_output(output):
- #Asked ChatGPT for this Regex (changed it to capture model responses that do not start with a numerical value)
- matched = re.search(r"([0-9]+(?:\.[0-9]*)?)", output)
+ #Asked ChatGPT for this Regex (changed it from original code to capture model responses that do not start with a numerical value)
+ matched = re.search(r"([1-5]+(?:\.[1-5]*)?)", output)
if matched:
- try:
- score = float(matched.group(1))
- except:
- score = 0
+ score = float(matched.group(1))
else:
score = 0
return score
@@ -115,6 +112,6 @@ if __name__ == '__main__':
d_ctr += 1
avg_pred_score = total_pred_score / scores_count
- print_correlations(results, n=d_ctr, input_file = input_file, dimension = dimension, avg_pred_score = avg_pred_score, output_file = output_file)
+ print_correlations(results, n=d_ctr, input_file = input_file, dimension = dimension, avg_pred_score = avg_pred_score, skipped_scores = skipped_scores, output_file = output_file)
else:
print(f"Skipping file: {input_file}")
\ No newline at end of file
diff --git a/prompts/cot_analysis/short_coh.txt b/prompts/cot_analysis/short_coh.txt
index e9939596b105902ec60bd190cde75e0ea352f0f3..c2e5e0eca4b863f10875a7ea4ad1c06eb0a9b729 100644
--- a/prompts/cot_analysis/short_coh.txt
+++ b/prompts/cot_analysis/short_coh.txt
@@ -10,11 +10,11 @@ Coherence (1-5) - the collective quality of all sentences. We align this dimensi
Evaluation Steps:
-1. Read.
-2. Assess quality.
-3. Determine if well-structured and well-organized.
-4. Decide if it forms a coherent body.
-5. Rate on scale of 1-5 for Coherence.
+1. read
+2. assess quality
+3. determine if structured/organized
+4. decide if coherent body
+5. rate on scale 1-5 for Coherence
Example:
diff --git a/prompts/cot_analysis/short_con.txt b/prompts/cot_analysis/short_con.txt
index fab2d80b2cb60a4c2a0e1e0272a049d17da3b7d9..712da4cd86c81d6b0b44917e2f2c41cab13ac261 100644
--- a/prompts/cot_analysis/short_con.txt
+++ b/prompts/cot_analysis/short_con.txt
@@ -10,10 +10,10 @@ Consistency (1-5) - the factual alignment between the summary and the summarized
Evaluation Steps:
-1. Read news article.
-2. Read summary.
-3. Identify inconsistencies/hallucinations.
-4. Rate on scale of 1-5 for Consistency.
+1. read news article
+2. read summary
+3. identify inconsistencies/hallucinations
+4. rate on scale 1-5 for Consistency
Example:
diff --git a/prompts/cot_analysis/short_rel.txt b/prompts/cot_analysis/short_rel.txt
index 53542265e0d4699ce80c94b9a7f3c5c11215da60..1cafa31b6b8723e2515c012705586124e91e4c61 100644
--- a/prompts/cot_analysis/short_rel.txt
+++ b/prompts/cot_analysis/short_rel.txt
@@ -10,11 +10,11 @@ Relevance (1-5) - selection of important content from the source. The summary sh
Evaluation Steps:
-1. Read.
-2. Identify important information.
-3. Determine if only important information included.
-4. Check redundancies/excess information.
-5. Rate on scale of 1-5 for Relevance.
+1. read
+2. identify important information
+3. determine if only important information included
+4. check redundancies/excess information
+5. rate on scale 1-5 for Relevance
Example:
diff --git a/prompts/cot_analysis/base_prompts.txt b/prompts/summeval/base_prompts.txt
similarity index 100%
rename from prompts/cot_analysis/base_prompts.txt
rename to prompts/summeval/base_prompts.txt
diff --git a/prompts/cot_analysis/detailed_prompts.txt b/prompts/summeval/detailed_prompts.txt
similarity index 100%
rename from prompts/cot_analysis/detailed_prompts.txt
rename to prompts/summeval/detailed_prompts.txt
diff --git a/qwen_eval.py b/qwen_eval.py
index 121e46882bc28425ae7a38c1bee66324a0f4bc68..0f9a8f8b523216297845257c91edc49d8ea5f8f5 100644
--- a/qwen_eval.py
+++ b/qwen_eval.py
@@ -5,7 +5,7 @@ import json
import time
import tqdm
-client = Groq(api_key="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")
+client = Groq(api_key=os.environ["GROQ_API_KEY"])
prompt_dir = 'prompts\\cot_analysis'
output_dir = 'results\\qwen'
@@ -20,8 +20,6 @@ sample_indices = sorted(random.sample(range(len(summeval)), 100))
sample_summeval = [summeval[i] for i in sample_indices]
for file in os.listdir(prompt_dir):
- if file.startswith(("gemini", "llama", "qwen")):
- continue
if file.endswith(".txt"):
prompt_fp = os.path.join(prompt_dir, file)
output_fp = os.path.join(output_dir, file.replace(".txt", ".json"))
@@ -41,7 +39,7 @@ for file in os.listdir(prompt_dir):
model=model,
messages=[{"role": "user", "content": cur_prompt}],
temperature=0,
- max_completion_tokens=5,
+ max_completion_tokens=10,
top_p=1,
frequency_penalty=0,
presence_penalty=0,
diff --git a/results/gemini/short_coh.json b/results/gemini/short_coh.json
index 6323fcd95d6aaad1ab14c17a2d7fcbf918faf5cc..89bd06739b438e1d336cf520fa81410df53290d9 100644
--- a/results/gemini/short_coh.json
+++ b/results/gemini/short_coh.json
@@ -12,8 +12,8 @@
"relevance": 2.3333333333333335,
"overall": 2.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
@@ -28,8 +28,8 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
@@ -44,7 +44,7 @@
"relevance": 3.3333333333333335,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -60,7 +60,7 @@
"relevance": 4.333333333333333,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -76,8 +76,8 @@
"relevance": 5.0,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
@@ -92,7 +92,7 @@
"relevance": 4.333333333333333,
"overall": 2.9166666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "2\n"
},
{
@@ -108,8 +108,8 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
@@ -124,7 +124,7 @@
"relevance": 5.0,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -140,7 +140,7 @@
"relevance": 3.6666666666666665,
"overall": 4.166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -156,8 +156,8 @@
"relevance": 4.0,
"overall": 4.333333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
@@ -172,8 +172,8 @@
"relevance": 4.666666666666667,
"overall": 4.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
@@ -188,8 +188,8 @@
"relevance": 2.6666666666666665,
"overall": 3.5833333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
@@ -204,7 +204,7 @@
"relevance": 3.3333333333333335,
"overall": 3.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -220,8 +220,8 @@
"relevance": 3.3333333333333335,
"overall": 3.7500000000000004
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
@@ -236,7 +236,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -252,8 +252,8 @@
"relevance": 2.0,
"overall": 2.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
@@ -268,8 +268,8 @@
"relevance": 3.3333333333333335,
"overall": 4.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
@@ -284,8 +284,8 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
@@ -300,7 +300,7 @@
"relevance": 3.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "2\n"
},
{
@@ -316,8 +316,8 @@
"relevance": 4.0,
"overall": 4.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
@@ -332,7 +332,7 @@
"relevance": 1.6666666666666667,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -348,8 +348,8 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
},
{
"doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
@@ -364,8 +364,8 @@
"relevance": 1.0,
"overall": 1.1666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1\n"
},
{
"doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
@@ -380,7 +380,7 @@
"relevance": 2.0,
"overall": 2.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -396,8 +396,8 @@
"relevance": 4.333333333333333,
"overall": 4.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
@@ -412,8 +412,8 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
@@ -428,8 +428,8 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
@@ -444,7 +444,7 @@
"relevance": 4.0,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -460,8 +460,8 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
@@ -476,7 +476,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -492,7 +492,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -508,7 +508,7 @@
"relevance": 4.0,
"overall": 4.083333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -524,7 +524,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -540,7 +540,7 @@
"relevance": 4.0,
"overall": 4.166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -556,8 +556,8 @@
"relevance": 4.333333333333333,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
@@ -572,8 +572,8 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
@@ -588,7 +588,7 @@
"relevance": 3.0,
"overall": 4.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -604,7 +604,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -620,7 +620,7 @@
"relevance": 3.0,
"overall": 2.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -636,8 +636,8 @@
"relevance": 2.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
@@ -652,7 +652,7 @@
"relevance": 4.666666666666667,
"overall": 4.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -668,8 +668,8 @@
"relevance": 3.0,
"overall": 2.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
@@ -684,8 +684,8 @@
"relevance": 3.0,
"overall": 2.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
},
{
"doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
@@ -700,7 +700,7 @@
"relevance": 3.0,
"overall": 3.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -716,7 +716,7 @@
"relevance": 3.3333333333333335,
"overall": 3.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -732,8 +732,8 @@
"relevance": 1.6666666666666667,
"overall": 1.4166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
},
{
"doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
@@ -748,7 +748,7 @@
"relevance": 3.0,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -764,7 +764,7 @@
"relevance": 4.0,
"overall": 4.083333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -780,8 +780,8 @@
"relevance": 2.0,
"overall": 3.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
@@ -796,7 +796,7 @@
"relevance": 4.666666666666667,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -812,7 +812,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -828,8 +828,8 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
@@ -844,7 +844,7 @@
"relevance": 4.333333333333333,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -860,8 +860,8 @@
"relevance": 2.3333333333333335,
"overall": 2.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
@@ -876,7 +876,7 @@
"relevance": 2.6666666666666665,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -892,7 +892,7 @@
"relevance": 2.6666666666666665,
"overall": 2.5833333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "2\n"
},
{
@@ -908,8 +908,8 @@
"relevance": 4.0,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
@@ -924,8 +924,8 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
@@ -940,8 +940,8 @@
"relevance": 2.3333333333333335,
"overall": 2.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
@@ -956,8 +956,8 @@
"relevance": 4.666666666666667,
"overall": 4.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
@@ -972,8 +972,8 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
@@ -988,8 +988,8 @@
"relevance": 3.6666666666666665,
"overall": 3.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
@@ -1004,7 +1004,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -1020,7 +1020,7 @@
"relevance": 3.3333333333333335,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1036,8 +1036,8 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
@@ -1052,8 +1052,8 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
@@ -1068,7 +1068,7 @@
"relevance": 4.0,
"overall": 4.166666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -1084,7 +1084,7 @@
"relevance": 4.333333333333333,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -1100,8 +1100,8 @@
"relevance": 4.0,
"overall": 4.416666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Coherence: "
},
{
"doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
@@ -1116,8 +1116,8 @@
"relevance": 3.0,
"overall": 3.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
@@ -1132,8 +1132,8 @@
"relevance": 3.6666666666666665,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
@@ -1148,7 +1148,7 @@
"relevance": 3.3333333333333335,
"overall": 3.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1164,8 +1164,8 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
@@ -1180,7 +1180,7 @@
"relevance": 3.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1196,7 +1196,7 @@
"relevance": 4.0,
"overall": 4.416666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -1212,7 +1212,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1228,8 +1228,8 @@
"relevance": 3.3333333333333335,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
@@ -1244,8 +1244,8 @@
"relevance": 4.0,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
@@ -1260,8 +1260,8 @@
"relevance": 1.6666666666666667,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Coherence: "
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
@@ -1276,8 +1276,8 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
@@ -1292,7 +1292,7 @@
"relevance": 4.0,
"overall": 3.9166666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1308,8 +1308,8 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
@@ -1324,7 +1324,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -1340,8 +1340,8 @@
"relevance": 4.333333333333333,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
@@ -1356,7 +1356,7 @@
"relevance": 4.0,
"overall": 4.166666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1372,8 +1372,8 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
@@ -1388,7 +1388,7 @@
"relevance": 4.0,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1404,8 +1404,8 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
@@ -1420,8 +1420,8 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
@@ -1436,7 +1436,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -1452,8 +1452,8 @@
"relevance": 2.6666666666666665,
"overall": 3.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
@@ -1468,7 +1468,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1484,8 +1484,8 @@
"relevance": 1.6666666666666667,
"overall": 3.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
@@ -1500,8 +1500,8 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
@@ -1516,7 +1516,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -1532,8 +1532,8 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
@@ -1548,8 +1548,8 @@
"relevance": 4.333333333333333,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
@@ -1564,8 +1564,8 @@
"relevance": 4.333333333333333,
"overall": 4.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
@@ -1580,7 +1580,7 @@
"relevance": 5.0,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1596,7 +1596,7 @@
"relevance": 3.0,
"overall": 3.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read.\n2. Assess quality.\n3. Determine if well-structured and well-organized.\n4. Decide if it forms a coherent body.\n5. Rate on scale of 1-5 for Coherence.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. read\n2. assess quality\n3. determine if structured/organized\n4. decide if coherent body\n5. rate on scale 1-5 for Coherence\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
}
]
\ No newline at end of file
diff --git a/results/gemini/short_con.json b/results/gemini/short_con.json
index 3a407660ae38664347778fb7fd4d843d1affbfff..0a9e9261382ac8c3b2313fd391ef2a00de180eb1 100644
--- a/results/gemini/short_con.json
+++ b/results/gemini/short_con.json
@@ -12,7 +12,7 @@
"relevance": 2.3333333333333335,
"overall": 2.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary: \n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary: \n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -28,7 +28,7 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -44,7 +44,7 @@
"relevance": 3.3333333333333335,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -60,7 +60,7 @@
"relevance": 4.333333333333333,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -76,7 +76,7 @@
"relevance": 5.0,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary: \n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary: \n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -92,8 +92,8 @@
"relevance": 4.333333333333333,
"overall": 2.9166666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1\n"
},
{
"doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
@@ -108,7 +108,7 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -124,7 +124,7 @@
"relevance": 5.0,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary: \n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary: \n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -140,7 +140,7 @@
"relevance": 3.6666666666666665,
"overall": 4.166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary: \n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary: \n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Consistency: 5"
},
{
@@ -156,7 +156,7 @@
"relevance": 4.0,
"overall": 4.333333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Consistency: 5"
},
{
@@ -172,8 +172,8 @@
"relevance": 4.666666666666667,
"overall": 4.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Consistency: 5"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
@@ -188,7 +188,7 @@
"relevance": 2.6666666666666665,
"overall": 3.5833333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Consistency: 5"
},
{
@@ -204,8 +204,8 @@
"relevance": 3.3333333333333335,
"overall": 3.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
@@ -220,8 +220,8 @@
"relevance": 3.3333333333333335,
"overall": 3.7500000000000004
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 5"
},
{
"doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
@@ -236,7 +236,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -252,7 +252,7 @@
"relevance": 2.0,
"overall": 2.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -268,7 +268,7 @@
"relevance": 3.3333333333333335,
"overall": 4.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary: \n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary: \n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -284,7 +284,7 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary: \n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary: \n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -300,7 +300,7 @@
"relevance": 3.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary: \n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary: \n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -316,7 +316,7 @@
"relevance": 4.0,
"overall": 4.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary: \n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary: \n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -332,7 +332,7 @@
"relevance": 1.6666666666666667,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary: \n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary: \n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -348,7 +348,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -364,7 +364,7 @@
"relevance": 1.0,
"overall": 1.1666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -380,7 +380,7 @@
"relevance": 2.0,
"overall": 2.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -396,7 +396,7 @@
"relevance": 4.333333333333333,
"overall": 4.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary: \n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary: \n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -412,7 +412,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary: \n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary: \n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -428,7 +428,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary: \n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary: \n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Consistency: 5"
},
{
@@ -444,7 +444,7 @@
"relevance": 4.0,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary: \n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary: \n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -460,7 +460,7 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -476,8 +476,8 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
@@ -492,7 +492,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -508,7 +508,7 @@
"relevance": 4.0,
"overall": 4.083333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -524,7 +524,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -540,8 +540,8 @@
"relevance": 4.0,
"overall": 4.166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
@@ -556,7 +556,7 @@
"relevance": 4.333333333333333,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -572,7 +572,7 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary: \n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary: \n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -588,7 +588,7 @@
"relevance": 3.0,
"overall": 4.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary: \n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary: \n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -604,7 +604,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -620,7 +620,7 @@
"relevance": 3.0,
"overall": 2.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -636,7 +636,7 @@
"relevance": 2.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -652,7 +652,7 @@
"relevance": 4.666666666666667,
"overall": 4.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -668,8 +668,8 @@
"relevance": 3.0,
"overall": 2.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 3"
},
{
"doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
@@ -684,8 +684,8 @@
"relevance": 3.0,
"overall": 2.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 3"
},
{
"doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
@@ -700,7 +700,7 @@
"relevance": 3.0,
"overall": 3.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -716,7 +716,7 @@
"relevance": 3.3333333333333335,
"overall": 3.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -732,8 +732,8 @@
"relevance": 1.6666666666666667,
"overall": 1.4166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary: \n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary: \n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
@@ -748,7 +748,7 @@
"relevance": 3.0,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -764,7 +764,7 @@
"relevance": 4.0,
"overall": 4.083333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -780,8 +780,8 @@
"relevance": 2.0,
"overall": 3.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 5"
},
{
"doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
@@ -796,7 +796,7 @@
"relevance": 4.666666666666667,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -812,7 +812,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary: \n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary: \n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -828,7 +828,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -844,7 +844,7 @@
"relevance": 4.333333333333333,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -860,8 +860,8 @@
"relevance": 2.3333333333333335,
"overall": 2.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
@@ -876,7 +876,7 @@
"relevance": 2.6666666666666665,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -892,7 +892,7 @@
"relevance": 2.6666666666666665,
"overall": 2.5833333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -908,7 +908,7 @@
"relevance": 4.0,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -924,7 +924,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary: \n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary: \n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "Okay, I understand."
},
{
@@ -940,7 +940,7 @@
"relevance": 2.3333333333333335,
"overall": 2.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "2\n"
},
{
@@ -956,7 +956,7 @@
"relevance": 4.666666666666667,
"overall": 4.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -972,7 +972,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary: \n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary: \n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -988,7 +988,7 @@
"relevance": 3.6666666666666665,
"overall": 3.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary: \n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary: \n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1004,7 +1004,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1020,7 +1020,7 @@
"relevance": 3.3333333333333335,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1036,7 +1036,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary: \n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary: \n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1052,7 +1052,7 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary: \n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary: \n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Consistency: 5"
},
{
@@ -1068,7 +1068,7 @@
"relevance": 4.0,
"overall": 4.166666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1084,7 +1084,7 @@
"relevance": 4.333333333333333,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1100,8 +1100,8 @@
"relevance": 4.0,
"overall": 4.416666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary: \n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Consistency: 5"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary: \n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 3"
},
{
"doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
@@ -1116,7 +1116,7 @@
"relevance": 3.0,
"overall": 3.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1132,8 +1132,8 @@
"relevance": 3.6666666666666665,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
@@ -1148,7 +1148,7 @@
"relevance": 3.3333333333333335,
"overall": 3.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1164,7 +1164,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1180,7 +1180,7 @@
"relevance": 3.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1196,8 +1196,8 @@
"relevance": 4.0,
"overall": 4.416666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary: \n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary: \n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 5"
},
{
"doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
@@ -1212,7 +1212,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary: \n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary: \n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1228,8 +1228,8 @@
"relevance": 3.3333333333333335,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 5"
},
{
"doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
@@ -1244,7 +1244,7 @@
"relevance": 4.0,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1260,7 +1260,7 @@
"relevance": 1.6666666666666667,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1276,7 +1276,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1292,8 +1292,8 @@
"relevance": 4.0,
"overall": 3.9166666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary: \n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary: \n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
@@ -1308,7 +1308,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1324,7 +1324,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary: \n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary: \n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1340,7 +1340,7 @@
"relevance": 4.333333333333333,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary: \n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary: \n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Consistency: 5"
},
{
@@ -1356,8 +1356,8 @@
"relevance": 4.0,
"overall": 4.166666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary: \n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary: \n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Consistency: 5"
},
{
"doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
@@ -1372,7 +1372,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary: \n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary: \n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1388,8 +1388,8 @@
"relevance": 4.0,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary: \n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "1\n\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary: \n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1\n"
},
{
"doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
@@ -1404,7 +1404,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1420,7 +1420,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1436,7 +1436,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1452,7 +1452,7 @@
"relevance": 2.6666666666666665,
"overall": 3.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary: \n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary: \n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1468,7 +1468,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary: \n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary: \n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1484,7 +1484,7 @@
"relevance": 1.6666666666666667,
"overall": 3.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1500,7 +1500,7 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1516,7 +1516,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1532,7 +1532,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1548,7 +1548,7 @@
"relevance": 4.333333333333333,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1564,7 +1564,7 @@
"relevance": 4.333333333333333,
"overall": 4.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1580,7 +1580,7 @@
"relevance": 5.0,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1596,7 +1596,7 @@
"relevance": 3.0,
"overall": 3.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read news article.\n2. Read summary.\n3. Identify inconsistencies/hallucinations.\n4. Rate on scale of 1-5 for Consistency.\n\n\nExample:\n\n\nSource Text: \n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary: \n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
- "model_response": "1\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. read news article\n2. read summary\n3. identify inconsistencies/hallucinations\n4. rate on scale 1-5 for Consistency\n\n\nExample:\n\n\nSource Text: \n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary: \n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
}
]
\ No newline at end of file
diff --git a/results/gemini/short_rel.json b/results/gemini/short_rel.json
index 08f6c1013c727774cdda8290bb7b935355fe2927..769b0214c72d62f71b44455b578bf42effc89110 100644
--- a/results/gemini/short_rel.json
+++ b/results/gemini/short_rel.json
@@ -12,7 +12,7 @@
"relevance": 2.3333333333333335,
"overall": 2.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -28,7 +28,7 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -44,8 +44,8 @@
"relevance": 3.3333333333333335,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
@@ -60,7 +60,7 @@
"relevance": 4.333333333333333,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -76,7 +76,7 @@
"relevance": 5.0,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -92,7 +92,7 @@
"relevance": 4.333333333333333,
"overall": 2.9166666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -108,8 +108,8 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
@@ -124,7 +124,7 @@
"relevance": 5.0,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -140,8 +140,8 @@
"relevance": 3.6666666666666665,
"overall": 4.166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Relevance: 5"
},
{
"doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
@@ -156,8 +156,8 @@
"relevance": 4.0,
"overall": 4.333333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Relevance: 5"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
@@ -172,7 +172,7 @@
"relevance": 4.666666666666667,
"overall": 4.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -188,8 +188,8 @@
"relevance": 2.6666666666666665,
"overall": 3.5833333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
@@ -204,7 +204,7 @@
"relevance": 3.3333333333333335,
"overall": 3.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -220,8 +220,8 @@
"relevance": 3.3333333333333335,
"overall": 3.7500000000000004
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
@@ -236,7 +236,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -252,7 +252,7 @@
"relevance": 2.0,
"overall": 2.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -268,8 +268,8 @@
"relevance": 3.3333333333333335,
"overall": 4.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
@@ -284,7 +284,7 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -300,8 +300,8 @@
"relevance": 3.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
@@ -316,7 +316,7 @@
"relevance": 4.0,
"overall": 4.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -332,7 +332,7 @@
"relevance": 1.6666666666666667,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -348,8 +348,8 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3Semantic annotations\n\nThere"
},
{
"doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
@@ -364,7 +364,7 @@
"relevance": 1.0,
"overall": 1.1666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -380,7 +380,7 @@
"relevance": 2.0,
"overall": 2.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -396,7 +396,7 @@
"relevance": 4.333333333333333,
"overall": 4.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -412,8 +412,8 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
@@ -428,7 +428,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Relevance: 5"
},
{
@@ -444,8 +444,8 @@
"relevance": 4.0,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
@@ -460,8 +460,8 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Relevance: 5"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
@@ -476,8 +476,8 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
@@ -492,7 +492,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -508,7 +508,7 @@
"relevance": 4.0,
"overall": 4.083333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -524,8 +524,8 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
@@ -540,8 +540,8 @@
"relevance": 4.0,
"overall": 4.166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
@@ -556,8 +556,8 @@
"relevance": 4.333333333333333,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Relevance: 5"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
@@ -572,7 +572,7 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -588,8 +588,8 @@
"relevance": 3.0,
"overall": 4.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Relevance: 4"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
@@ -604,7 +604,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -620,8 +620,8 @@
"relevance": 3.0,
"overall": 2.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "1\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1 \n"
},
{
"doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
@@ -636,8 +636,8 @@
"relevance": 2.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1\n"
},
{
"doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
@@ -652,7 +652,7 @@
"relevance": 4.666666666666667,
"overall": 4.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -668,8 +668,8 @@
"relevance": 3.0,
"overall": 2.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "1\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
@@ -684,7 +684,7 @@
"relevance": 3.0,
"overall": 2.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -700,8 +700,8 @@
"relevance": 3.0,
"overall": 3.75
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
},
{
"doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
@@ -716,7 +716,7 @@
"relevance": 3.3333333333333335,
"overall": 3.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "1\n"
},
{
@@ -732,7 +732,7 @@
"relevance": 1.6666666666666667,
"overall": 1.4166666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -748,7 +748,7 @@
"relevance": 3.0,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -764,7 +764,7 @@
"relevance": 4.0,
"overall": 4.083333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "3\n"
},
{
@@ -780,8 +780,8 @@
"relevance": 2.0,
"overall": 3.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
},
{
"doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
@@ -796,7 +796,7 @@
"relevance": 4.666666666666667,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -812,7 +812,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -828,8 +828,8 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
@@ -844,7 +844,7 @@
"relevance": 4.333333333333333,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
},
{
@@ -860,7 +860,7 @@
"relevance": 2.3333333333333335,
"overall": 2.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -876,7 +876,7 @@
"relevance": 2.6666666666666665,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -892,7 +892,7 @@
"relevance": 2.6666666666666665,
"overall": 2.5833333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "2\n"
},
{
@@ -908,8 +908,8 @@
"relevance": 4.0,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
@@ -924,7 +924,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "- Relevance: 5"
},
{
@@ -940,8 +940,8 @@
"relevance": 2.3333333333333335,
"overall": 2.666666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2\n"
},
{
"doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
@@ -956,7 +956,7 @@
"relevance": 4.666666666666667,
"overall": 4.916666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -972,7 +972,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -988,8 +988,8 @@
"relevance": 3.6666666666666665,
"overall": 3.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
@@ -1004,7 +1004,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1020,8 +1020,8 @@
"relevance": 3.3333333333333335,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
@@ -1036,7 +1036,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1052,7 +1052,7 @@
"relevance": 4.0,
"overall": 4.5
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1068,7 +1068,7 @@
"relevance": 4.0,
"overall": 4.166666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1084,8 +1084,8 @@
"relevance": 4.333333333333333,
"overall": 4.25
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
@@ -1100,8 +1100,8 @@
"relevance": 4.0,
"overall": 4.416666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Relevance: 3"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "- Relevance: 4"
},
{
"doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
@@ -1116,8 +1116,8 @@
"relevance": 3.0,
"overall": 3.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
@@ -1132,8 +1132,8 @@
"relevance": 3.6666666666666665,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
@@ -1148,7 +1148,7 @@
"relevance": 3.3333333333333335,
"overall": 3.8333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1164,7 +1164,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1180,7 +1180,7 @@
"relevance": 3.6666666666666665,
"overall": 3.6666666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1196,7 +1196,7 @@
"relevance": 4.0,
"overall": 4.416666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1212,7 +1212,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1228,7 +1228,7 @@
"relevance": 3.3333333333333335,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1244,8 +1244,8 @@
"relevance": 4.0,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
@@ -1260,8 +1260,8 @@
"relevance": 1.6666666666666667,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
@@ -1276,7 +1276,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1292,8 +1292,8 @@
"relevance": 4.0,
"overall": 3.9166666666666665
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "2\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1\n"
},
{
"doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
@@ -1308,7 +1308,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1324,7 +1324,7 @@
"relevance": 3.6666666666666665,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1340,8 +1340,8 @@
"relevance": 4.333333333333333,
"overall": 4.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "- Relevance: 5"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
@@ -1356,8 +1356,8 @@
"relevance": 4.0,
"overall": 4.166666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "3\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
@@ -1372,7 +1372,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1388,7 +1388,7 @@
"relevance": 4.0,
"overall": 3.3333333333333335
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1404,7 +1404,7 @@
"relevance": 4.333333333333333,
"overall": 4.666666666666666
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1420,8 +1420,8 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
},
{
"doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
@@ -1436,7 +1436,7 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1452,8 +1452,8 @@
"relevance": 2.6666666666666665,
"overall": 3.833333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
@@ -1468,8 +1468,8 @@
"relevance": 4.333333333333333,
"overall": 4.333333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "4\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
},
{
"doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
@@ -1484,8 +1484,8 @@
"relevance": 1.6666666666666667,
"overall": 3.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
- "model_response": "5\n"
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4\n"
},
{
"doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
@@ -1500,7 +1500,7 @@
"relevance": 4.666666666666667,
"overall": 4.833333333333334
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1516,7 +1516,7 @@
"relevance": 4.333333333333333,
"overall": 4.583333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1532,7 +1532,7 @@
"relevance": 5.0,
"overall": 5.0
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1548,7 +1548,7 @@
"relevance": 4.333333333333333,
"overall": 4.083333333333333
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "5\n"
},
{
@@ -1564,7 +1564,39 @@
"relevance": 4.333333333333333,
"overall": 4.416666666666667
},
- "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read.\n2. Identify important information.\n3. Determine if only important information included.\n4. Check redundancies/excess information.\n5. Rate on scale of 1-5 for Relevance.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
"model_response": "4\n"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5\n"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. read\n2. identify important information\n3. determine if only important information included\n4. check redundancies/excess information\n5. rate on scale 1-5 for Relevance\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3\n"
}
]
\ No newline at end of file
diff --git a/results/llama/gemini_coh_detailed.json b/results/llama/gemini_coh_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..01a8402256186f392830a2ca829e722cd96d008b
--- /dev/null
+++ b/results/llama/gemini_coh_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the entire summary to get a general understanding of its content.\n2. **Identify the Main Topic(s):** Determine the central theme(s) or event(s) the summary is trying to convey.\n3. **Analyze Sentence-to-Sentence Flow:** Evaluate how each sentence connects to the preceding and following sentences. Look for logical connections, transitions, and a clear progression of ideas.\n4. **Assess Overall Structure:** Determine if the summary has a discernible structure (e.g., chronological, cause-and-effect, problem-solution). Is there a clear beginning, middle, and end (even if brief)?\n5. **Identify Disruptions:** Look for any instances where the flow is disrupted, such as abrupt topic changes, missing information, or illogical connections.\n6. **Assign a Coherence Score (1-5):**\n * **1 (Very Poor):** The summary is a collection of disconnected sentences with no clear structure or logical flow. It's difficult to understand the main topic.\n * **2 (Poor):** The summary has some related information, but the sentences are poorly connected and the overall structure is weak.\n * **3 (Fair):** The summary presents a somewhat coherent body of information, but the structure could be improved, and some connections between sentences are weak.\n * **4 (Good):** The summary is well-structured and organized, with clear connections between sentences. It presents a coherent body of information about the topic.\n * **5 (Excellent):** The summary is exceptionally well-structured and organized, with seamless transitions and a logical flow of ideas. It presents a highly coherent and easily understandable account of the topic.\n7. **Justify the Score (Optional but Recommended):** Briefly explain why you assigned the chosen score, highlighting specific strengths or weaknesses in the summary's coherence.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/gemini_con_detailed.json b/results/llama/gemini_con_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..37c47f9dc531eda5411bc3d3b735c6e14a8d967c
--- /dev/null
+++ b/results/llama/gemini_con_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary: \n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary: \n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary: \n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary: \n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary: \n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary: \n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary: \n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary: \n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary: \n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary: \n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary: \n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary: \n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary: \n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary: \n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary: \n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary: \n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary: \n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary: \n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary: \n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary: \n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary: \n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary: \n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary: \n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary: \n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary: \n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary: \n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary: \n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary: \n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary: \n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary: \n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary: \n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary: \n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary: \n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. **Read the Source Article Carefully:** Understand the main points, key details, and overall context of the news article.\n2. **Read the Summary Carefully:** Identify all factual claims made in the summary.\n3. **Verify Each Claim:** For each factual claim in the summary, check if it is explicitly stated or directly entailed by the source article.\n4. **Identify Inconsistencies:** Note any claims in the summary that are:\n * **Not Supported by the Source:** The source article does not mention or imply the claim.\n * **Contradicted by the Source:** The source article states something that contradicts the claim.\n * **Hallucinated:** The claim is completely fabricated and has no basis in the source article.\n5. **Assign a Consistency Score (1-5):**\n * **5 - Perfectly Consistent:** All claims in the summary are factually accurate and supported by the source article.\n * **4 - Mostly Consistent:** The summary is largely accurate, with only minor inconsistencies or omissions that do not significantly alter the meaning.\n * **3 - Partially Consistent:** The summary contains some accurate information, but also includes noticeable inconsistencies or unsupported claims.\n * **2 - Mostly Inconsistent:** The summary contains more inaccurate or unsupported claims than accurate ones.\n * **1 - Completely Inconsistent:** The summary is entirely fabricated or bears little to no resemblance to the source article.\n6. **Justify the Score:** Briefly explain *why* you assigned the chosen score, highlighting specific examples of consistent or inconsistent claims. Mention if hallucinated facts are present.\n\n\nExample:\n\n\nSource Text: \n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary: \n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/gemini_rel_detailed.json b/results/llama/gemini_rel_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..f72e8c28fb1465e473745860791d1e7e805df2ef
--- /dev/null
+++ b/results/llama/gemini_rel_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. **Read the Summary:** Carefully read the provided summary.\n2. **Identify Key Information:** Determine the most important facts, events, and figures presented in the summary.\n3. **Consider the Source (Mentally):** Imagine you've read the original news article. What information *must* be included in a good summary?\n4. **Assess Importance:** Does the summary include the *most* important information from the source? Is anything crucial missing?\n5. **Identify Redundancy/Excess:** Does the summary contain any information that is repetitive, unnecessary, or overly detailed for a brief overview?\n6. **Assign a Relevance Score (1-5):**\n * **5 - Highly Relevant:** The summary includes all the most important information and excludes unnecessary details. It's a concise and accurate representation of the core content.\n * **4 - Mostly Relevant:** The summary includes most of the important information, but might be missing a minor detail or contain a very small amount of unnecessary information.\n * **3 - Moderately Relevant:** The summary includes some important information, but is missing significant details or contains a noticeable amount of unnecessary information.\n * **2 - Somewhat Relevant:** The summary includes only a few important details and is missing a lot of crucial information, or is overwhelmed by unnecessary information.\n * **1 - Not Relevant:** The summary fails to capture the main points of the article or is entirely focused on irrelevant details.\n7. **Provide Justification (Optional, but helpful):** Briefly explain *why* you assigned the score you did. Mention specific missing details or instances of redundancy.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/llama_coh_detailed.json b/results/llama/llama_coh_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..3183e0facb2ed050513b0084b17e9d7f1cc49b60
--- /dev/null
+++ b/results/llama/llama_coh_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Assess the collective quality of all sentences in the summary.\n3. Determine if the summary is well-structured and well-organized.\n4. Decide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\n5. Rate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/llama_con_detailed.json b/results/llama/llama_con_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..58c6df3a3a317f9ee1f9c62c5ed891c40ecb88c5
--- /dev/null
+++ b/results/llama/llama_con_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary: \n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary: \n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary: \n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary: \n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary: \n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary: \n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary: \n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary: \n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary: \n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary: \n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary: \n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary: \n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary: \n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary: \n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary: \n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary: \n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary: \n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary: \n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary: \n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary: \n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary: \n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary: \n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary: \n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary: \n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary: \n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary: \n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary: \n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary: \n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary: \n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary: \n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary: \n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary: \n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary: \n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article carefully to understand its content.\n2. Read the provided summary of the article.\n3. Compare the summary with the original article to identify any inconsistencies or hallucinated facts.\n4. Rate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\n5. Assign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\n\n\nExample:\n\n\nSource Text: \n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary: \n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/llama_rel_detailed.json b/results/llama/llama_rel_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb454846650bdac7f82affb822bb2058c5c169e1
--- /dev/null
+++ b/results/llama/llama_rel_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the provided summary carefully.\n2. Identify the important information from the source document (implied, not provided).\n3. Determine if the summary includes only the important information from the source document.\n4. Check for redundancies and excess information in the summary.\n5. Rate the summary on a scale of 1-5 for Relevance, based on the selection of important content from the source.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/qwen_coh_detailed.json b/results/llama/qwen_coh_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..c8f19278b79aff65332a426f93d7dbde05642fe9
--- /dev/null
+++ b/results/llama/qwen_coh_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main topic and the information presented.\n2. Assess the logical flow of information from one sentence to the next, ensuring that each piece of information builds upon or connects to the previous one.\n3. Check for the presence of a clear introduction that sets the context and a conclusion that wraps up the main points.\n4. Identify if there are any abrupt shifts in the information that disrupt the flow or if the sentences are well-connected, contributing to a cohesive narrative.\n5. Assign a score from 1 to 5 based on the overall coherence of the summary, where 1 indicates a poorly structured summary with disjointed information and 5 indicates a well-structured and logically coherent summary.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/qwen_con_detailed.json b/results/llama/qwen_con_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..46df46f189e0cd7c9c1a380ca66ff6e5877ac7a9
--- /dev/null
+++ b/results/llama/qwen_con_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary: \n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary: \n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary: \n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary: \n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary: \n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary: \n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary: \n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary: \n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary: \n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary: \n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary: \n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary: \n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary: \n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary: \n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary: \n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary: \n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary: \n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary: \n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary: \n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary: \n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary: \n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary: \n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary: \n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary: \n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary: \n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary: \n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary: \n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary: \n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary: \n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary: \n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary: \n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary: \n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary: \n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary: \n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary: \n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary: \n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary: \n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary: \n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary: \n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary: \n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary: \n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary: \n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary: \n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary: \n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary: \n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary: \n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary: \n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary: \n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary: \n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary: \n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary: \n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary: \n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary: \n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary: \n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary: \n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary: \n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\n1. Read the news article thoroughly to understand the main points, details, and context provided.\n2. Read the summary carefully, noting the key points and information presented.\n3. Compare the summary to the article, checking if all the information in the summary is directly supported by the content of the article.\n4. Identify any facts or details in the summary that are not present or not supported by the article, which would indicate inconsistency or hallucination.\n5. Rate the summary on a scale of 1 to 5 based on the Consistency criteria, where 1 indicates significant inconsistency or presence of hallucinated facts, and 5 indicates perfect alignment with the source article without any inconsistencies or hallucinations.\n6. Justify the rating by pointing out specific examples from the summary that support the given score.\n\n\nExample:\n\n\nSource Text: \n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary: \n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/qwen_rel_detailed.json b/results/llama/qwen_rel_detailed.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ba4da49fe4231f2b38f96f5a6bb1eb47ebcc6b3
--- /dev/null
+++ b/results/llama/qwen_rel_detailed.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "1"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nRelevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.\n\nEvaluation Steps:\n\n1. Read the summary thoroughly to understand the main points it covers.\n2. Compare the summary's content with the original news article to identify if all the key information from the article is included.\n3. Assess whether the summary avoids unnecessary details or redundancies that do not contribute to the core message of the article.\n4. Assign a score from 1 to 5 based on how well the summary adheres to the criteria of relevance, where 1 indicates the summary is not relevant at all and 5 indicates the summary is highly relevant, including only the most important information without any excess details.\n5. Note any specific examples in the summary that support your rating, such as inclusion of key facts or exclusion of irrelevant information.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Relevance: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/scrambled_coh.json b/results/llama/scrambled_coh.json
new file mode 100644
index 0000000000000000000000000000000000000000..bc5f50b117903468787c0cc67ae33491fa95c225
--- /dev/null
+++ b/results/llama/scrambled_coh.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary:\n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary:\n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary:\n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary:\n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary:\n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary:\n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary:\n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary:\n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary:\n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary:\n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary:\n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary:\n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '\n\nSummary:\n\nElla henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-d75b043ebefc3098aea84d92bb8bec0f509b1563",
+ "system_id": "M10",
+ "source": "( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .",
+ "reference": "The Pakistani government says its security agencies are investigating . A group believed to be based in Pakistan 's Balochistan province claims responsibility .",
+ "system_output": "Three of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Eight Iranian border guards have been killed in clashes with militants near the border with Pakistan , Iranian state media reported . Three of the militants were killed by Iranian forces in the fighting Monday in the southeastern town of Negur , the state-run news agency IRNA reported . The news agency cited Ali Asghar Mirshekari , the deputy governor of Iran 's Sistan-Baluchestan province , who said the militants crossed into the country from Pakistan . Iranian officials have reportedly asked Pakistani authorities to catch the surviving assailants . A militant group called Jaish al Adal claimed responsibility for the attack , according to Iranian state media and a Twitter account believed to be associated with the organization . According to Pakistani media , Jaish al Adal is an anti-Iranian group that has emerged during recent years in areas of Balochistan , the restive Pakistani province that borders southeastern Iran . The Sunni Muslim group has targeted Shiites and says it aims to thwart Iranian influence in Pakistan , according to the Pakistani newspaper Dawn . Jaish al Adal has also claimed responsibility for attacks on Iranian territory . One of the deadliest was in October 2013 , when 14 Iranian border guards were killed near the city of Saravan . Iranian authorities say that after similar attacks , the militants have fled back into Balochistan . `` Iran has repeatedly criticized its eastern neighbor for failing to rein in the terrorists , '' IRNA said , referring to Pakistan . The Pakistani government condemned Monday 's attack , calling it an `` act of terrorism . '' `` The concerned security agencies of Pakistan are actively investigating this incident , which took place on Iran 's territory , and have asked Iranian authorities to share with us any evidence that they have , '' the government said in a statement Wednesday . If the militants entered Pakistan after the attack , `` they will be apprehended and brought to justice , '' it said .\n\nSummary:\n\nThree of the militants were killed by iranian forces near the border with pakistan , iranian state media reports . Jaish al adal says it aims to thwart iranian influence in pakistan . The sunni muslim group has targeted shiites .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M5",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n( cnn ) the classic video game `` space invaders `` was developed in japan back in the late 1970 's -- and now their real - life counterparts are the topic of an earnest political discussion in japan 's corridors of power . Luckily , japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's air self defense force ( asdf ) had never encountered an extraterrestrial unidentified flying object .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M8",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.0,
+ "fluency": 1.3333333333333333,
+ "relevance": 1.0,
+ "overall": 1.1666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\n`` space invaders `` is the founder of the new japan pro wrestling organization . Inoki has appeared in the u.s.-based wwe -- which describes him as `` among the most respected men in sports-entertainment `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e2bccd4dec93c9bb7b327827dae004c2d494ec31",
+ "system_id": "M10",
+ "source": "( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II",
+ "reference": "Japan 's top military official earnestly revealed that the country 's Self Defense Force ( SDF ) had never encountered a UFO . Celebrity politician and former wrestler Antonio Inoki had posed a question concerning extraterrestrials to a government committee .",
+ "system_output": "The classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 2.6666666666666665,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The classic video game `` Space Invaders '' was developed in Japan back in the late 1970 's -- and now their real-life counterparts are the topic of an earnest political discussion in Japan 's corridors of power . Luckily , Japanese can sleep soundly in their beds tonight as the government 's top military official earnestly revealed that the country 's Air Self Defense Force ( ASDF ) had never encountered an extraterrestrial unidentified flying object . Responding to a query from flamboyant former wrestler-turned-lawmaker Antonio Inoki , Defense Minister Gen Nakatani told the Diet , Japan 's parliament , that his jets had , to date , never come across any UFOs from outer space . `` When the Air Self Defense Force detects indications of an unidentified flying object that could violate our country 's airspace , it scrambles fighter jets if necessary and makes visual observation , '' Nakatani said . He continued : `` They sometimes find birds or flying objects other than aircraft but I do n't know of a case of finding an unidentified flying object believed to have come over from anywhere other than Earth . '' Inoki has appeared in the U.S.-based WWE -- which describes him as `` among the most respected men in sports-entertainment '' -- and is the founder of the New Japan Pro Wrestling organization . He entered Japan 's Upper House for a second stint in politics in 2013 . He also famously fought Muhammad Ali in 1976 , in one of the first-ever mixed-discipline matches , which would later pave the way for today 's wildly popular Mixed Martial Arts contests . Before his return to politics he was a regular fixture on Japanese TV variety shows and has promoted a slew of products , from hot sauce to banks . The maverick politician also traveled to Iraq in 1990 to try to secure the release of Japanese hostages , and has more recently attempted to replicate former NBA star Dennis Rodman 's `` basketball diplomacy '' by staging a wrestling tournament in North Korea . He reportedly converted to Islam in the 1990s , although he says he practices both Islam and Buddhism . The lawmaker , who is universally known in Japan for his colossal chin and once-ever-present red scarf -- these days often replaced with a red necktie -- as much as for his political achievements , had asked a Upper House Budget Committee meeting if aircraft were ever scrambled to meet extraterrestrial threats , and if research was being done into alien visitors , prompting Nakatani 's response . Inoki also claims to have seen a UFO with his own eyes , but admitted that he did n't know personally if aliens existed . The exchange was n't the first time Japanese politicians have discussed the implications of visitors from another planet . In 2007 then-Defense Minister Shigeru Ishiba pondered the legal ramifications , under Japan 's pacifist constitution , of a defense against an invasion from outer space . READ MORE : Japan unveils Izumo , its largest warship since World War II\n\nSummary:\n\nThe classic video game `` space invaders `` was developed in japan . Defense minister gen nakatani said japan 's air self defense force never had an extraterrestrial unidentified flying object . Inoki has appeared in the u.s.-based wwe .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-54b857a66cbd8473d6732e5da52d48612e636a37",
+ "system_id": "M10",
+ "source": "A passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '",
+ "reference": "Jin Pai was standing on rim of a toilet in Hefei Xinqiao International Airport . The porcelain toilet then tipped over and shattered on the floor . The 35-year-old is left with deep cuts to his leg and buttocks .",
+ "system_output": "Passenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA passenger waiting to board a plane in an airport terminal has been hospitalised with deep cuts on his leg and buttocks after he broke a toilet he was squatting on . Passenger Jin Pai , 35 , was standing on the rim of a toilet in Hefei Xinqiao International Airport in the city of Hefei , the capital city of eastern China 's Anhui Province , when it smashed to the ground . According to airport officials he had not wanted to let his bottom touch the seat because he was 'worried it might not be clean ' . Passenger Jin Pai , 35 , was standing on the rim of a toilet when it collapsed , leaving him hospitalised The smashed up toilet in Hefei Xinqiao International Airport in the city of Hefei But he was caught out when the porcelain toilet then tipped over and shattered on the floor , cutting the man in several places . The injured man was found bleeding on the toilet floor after he had telephoned his travelling companions for help . He had been too embarrassed to shout for help and had hoped they would assist but he was too badly hurt for anything other than an immediate trip to hospital . Authorities and emergency personnel arrived to check the man into a Hefei hospital , where he received several dozens stitches on his left leg and buttocks . Then man broke the toilet in Hefei Xinqiao International Airport in the city of Hefei The passenger reportedly confessed to his friends that he was squatting on the toilet , but later changed his story to the police , possibly to avoid paying compensation , and said he was sitting when the poorly built toilet gave way . Authorities said they believed the incident was an accident . Police spokesman Bo Chiang said : 'In any case , we would like to remind people that these toilets are for sitting on , not for standing on . '\n\nSummary:\n\nPassenger jin pai , 35 , was standing on the rim of a toilet in hefei , china 's anhui province , when it smashed to the ground . He was caught out when the toilet then tipped over and shattered on the toilet floor . He had been too embarrassed to shout for help but he changed his story to the police .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5bc2ae78441e4ac08fb01823d5fc0f1627c3166",
+ "system_id": "M23",
+ "source": "The dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey",
+ "reference": "Kevin Pietersen lined up with Surrey team-mates for team photo . Batsman was in a jovial mood as he joked with photographers at KIA Oval . Pietersen could be in line for a return to England Test team this summer . Paul Downton dismissal as managing director has opened door .",
+ "system_output": "Kevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThe dismissal of Paul Downton as England 's managing director may be the big talking point in English cricket but focus has already shifted to the start of the county championship season . However , you do n't have to go to far before talk returns to the England team again as Kevin Pietersen cut a range of emotions as he posed for the Surrey team photo before the start of the new campaign . Pietersen looked in deep thought as he sat right at the end of the team photo , but was otherwise in a jovial mood as a potentially sensational return to the England set-up took a step closer . Kevin Pietersen takes a moment to reflect ahead of a team photo shoot for Surrey The event at the KIA Oval comes before the start of the County Championship season As team-mates turn their attention to other matters , Pietersen looks ahead in deep thought The batsman joked with photographers at the KIA Oval for the photo session , with Surrey set to begin their campaign at Glamorgan on April 19 . Downton 's sacking on Wednesday has paved the way for the 34-year-old to return to the international set-up having been controversially axed by the former managing director in 2014 . It was during the last Ashes tour for England in Australia last year when Pietersen last featured at international level , in a disastrous trip which saw his side suffer a whitewash series defeat . With England hoping to regain the urn this summer on home soil , Pietersen has rejoined Surrey in an attempt to force his way back into the Test set-up . Surrey play Oxford MCCU in a MCC University match on Sunday . Kevin Pietersen took his place on the right of the front row for the Surrey team photo shoot at the KIA Oval Using his camera , Pietersen turned the tables to take pictures of photographers Team-mate Chris Tremlett flashes a smile as he and Pietersen share a joke during the event Photographers gather round Pietersen to get close-up pictures of the batsman Pietersen flashes a cheeky grin to the camera as he sat at the end of the front row for the team photo Pietersen was all smiles as he joked with team-mates ahead of the new county championship season Pietersen smiles as he sits alongside Chris Tremlett ( second right ) , Stuart Meaker ( centre ) , Jason Roy ( second left ) and Jade Dernbach Pietersen walks out to the KIA Oval pitch with a tracksuit top on , closely followed by his Surrey team-mates Pietersen jokes with a photographer during the Surrey CCC photocall at The Kia Oval Pietersen looks on from inside the KIA Oval as he hopes to work his way back into the England set-up with Surrey\n\nSummary:\n\nKevin pietersen posed for the surrey team photo at the kia oval . The event comes before the start of the county championship season . Surrey begin their campaign at glamorgan on april 19 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-130ff8f2dfb6ab67861d5924b0ff5acd63c7d8ce",
+ "system_id": "M17",
+ "source": "Relegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .",
+ "reference": "Romanian club Ceahlaul Piatra Neamt sacked Ze Maria on Wednesday . But the former Brazil international was reinstated the next day . He was then sacked again following a defeat on Saturday .",
+ "system_output": "Ceahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nRelegation-threatened Romanian club Ceahlaul Piatra Neamt have sacked Brazilian coach Ze Maria for the second time in a week . Former Brazil defender Ze Maria was fired on Wednesday after a poor run , only to be reinstated the next day after flamboyant owner Angelo Massone decided to 'give the coaching staff another chance . ' But the 41-year-old former Inter Milan and Parma right back , capped 25 times by Brazil , angered Massone again after Ceahlaul were beaten 2-0 by mid-table FC Botosani on Saturday . Ze Maria represented Brazil on 25 occasions during an international career spanning five years The result left Ceahlaul 16th in the standings , six points adrift of safety . Ze Maria replaced Florin Marin in January to become Ceahlaul 's third coach this season . He will be replaced by Serbian Vanya Radinovic .\n\nSummary:\n\nCeahlaul piatra neamt have sacked brazilian coach ze maria for the second time in a week . Ze maria was fired on wednesday after a poor run , only to be reinstated the next day . The 41-year-old angered owner angelo massone again after ceahlaul were beaten 2-0 by mid-table fc botosani on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4001b252a072ac149c70840b22299cc6cfab3bae",
+ "system_id": "M8",
+ "source": "Radamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '",
+ "reference": "Radamel Falcao joined Manchester United on on loan last summer . Striker has only managed four goals in a disappointing season . Nabil Dirar said he would have been better off saying at Monaco .",
+ "system_output": "Radamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nRadamel Falcao 's former Monaco team-mate Nabil Dirar believes the Colombian must be regretting his failed loan move to Manchester United even though he was not happy in France . The former Atletico Madrid striker moved to Old Trafford on a season-long loan last summer but has struggled to hit the heights he achieved before suffering a serious knee injury . United will have to pay \u00a346million to make Falcao 's transfer permanent but having only scored four goals all season Louis van Gaal 's side are unlikely to take up that option . Radamel Falcao has struggled for Manchester United during his season-long loan Falcao has struggled to reach the heights he achieved before suffering a serious knee injury And Dirar has revealed that Falcao was not happy in France but thinks he would have been better off staying at Monaco . ' I think Falcao must be biting his fingers [ ruing the fact ] he 's gone , ' he told Le Parisien . 'Here , he would have played and participated in the Champions League . But he may need more atmosphere . 'In Monaco , he was not feeling well . '\n\nSummary:\n\nRadamel falcao 's former monaco team-mate nabil dirar believes the colombian must be regretting his failed loan move to manchester united even though he was not happy in france . The former atletico madrid striker moved to old trafford on a season-long loan last summer . Falcao has struggled to reach the heights he achieved before suffering a serious knee injury .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M20",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-cf4ba8f94f2dd5604eb8f8a4e655c10cf0996365",
+ "system_id": "M9",
+ "source": "New Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .",
+ "reference": "Police were called to a dairy in east Christchurch after reports of a robbery . Man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter . Police are appealing to the public for help in identifying the offender .",
+ "system_output": "New zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nNew Zealand police are appealing to the public to help identify a man who robbed a Christchurch dairy wearing a 'distinctive ' cartoon mask . The offender , snapped on security footage in a fluoro orange sweatshirt and oversized mask , is described as being 'very tall ' . He made off with the dairy 's till and about $ 1500 in cash on Saturday , Stuff.co.nz reports . A man in a 'distinctive ' cartoon mask demanded cash from owner 's daughter on Saturday Police were reportedly called to the Union St dairy , in east Christchurch at about 12.30pm , after receiving reports of a masked man demanding cash from the dairy owner 's daughter . Christina Chen told reporters her 30-year-old daughter ran out of the shop screaming 'robber ! ' . The masked man threw the cash register to the ground and took off with the till drawer and up to $ 1,500 . 'Normally they just rob cigarettes and some more things , it 's not a big issue , ' Chen told Stuff.co.nz . 'But this time they were scary because they wore a mask , and we do n't know who it is . ' Police say there were no signs of any weapons used but that the robber remained at large . They are asking anyone with information that may help to identify this person to call Crimestoppers on 0800 555 111 or to message police online .\n\nSummary:\n\nNew zealand police are appealing to the public to help identify a man who robbed a christchurch dairy . He made off with the dairy 's till and about $ 1500 in cash . Man in ` distinctive ' cartoon mask demanded cash from owner 's daughter . Police were reportedly called to the union st dairy , in east christchurch .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M13",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , accused of having sex with pupil , 15 , in her car on consecutive nights in january . She was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also accused granado of having sex with him on two consecutive nights in january this year .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M14",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , a teacher at macarthur high school in irving , was first arrested on march 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused granado of having sex with him on two consecutive nights in january . Granado was arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8e6c8aec6808f66ed0342e128a4e146f3ecd627d",
+ "system_id": "M23",
+ "source": "Kaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '",
+ "reference": "Kaitlyn Granado , 24 , first arrested March 19 for relationship with boy , 15 . Admitting kissing and letting him touch her and was released on bail . Officers quizzed another 15-year-old on March 24 over second relationship . Granado arrested again amid claims she had sex with second boy in car .",
+ "system_output": "Kaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nKaitlyn Granado , 24 , was arrested on Monday amid claims she had sex with pupil , 15 , in her car on consecutive nights in January A Texas math teacher has been accused of having sex with an underage pupil from a Texas high school a month after she was arrested for letting another teenage touch her breasts . Kaitlyn Granado , 24 , a teacher at MacArthur High School in Irving , was first arrested on March 19 for having an inappropriate relationship with a 15-year-old student , and was released on bail . Now she has been arrested again after another pupil , also aged 15 , also accused Granado of having sex with him on two consecutive nights in January this year . According to the affidavit , seen by the Star-Telegram , the teen claims the sexual relationship took place in her car late at night on January 18 and 19 across the road from another school in Dallas . The abuse came to light after a school police officer learned of a possible relationship between Granado and one of the boys last month . She was arrested on March 19 , and in a police interview she admitted kissing the pupil and allowing him to touch her breasts in April the previous year , according to the Dallas Morning News . She was subsequently suspended from teaching and put on administrative leave by school officials . At the time , detectives said they did n't think she had been in a relationship with any other pupils , but just days later , on March 24 , officers quizzed a second boy about a possible relationship with her . A spokesman for the police department said : 'There was some discussion , some rumoring , there may have been a second person involved . 'That information came from within the school , and it turned out to be true . ' Granado , who is a maths teacher at MacArthur High School in Irving , Texas ( pictured ) , was also arrested last month when she admitted kissing another boy , also 15 , and lettin him touch her breasts Granado was booked into Irving County Jail for the second time on Monday , and released after posting a $ 50,000 bail . It is not known how Granado intends to plead to the second set of charges against her . A spokesman for Irving school district said : 'While privacy laws prevent us from sharing specific details , we can reassure our families that we took immediate and appropriate action . 'Being on administrative leave prevents her from working with students and being present on any Irving ISD property . '\n\nSummary:\n\nKaitlyn granado , 24 , is a maths teacher at macarthur high school in irving . She was first arrested on march 19 for having inappropriate relationship . In police interview she admitted kissing 15-year-old and letting him touch breasts . She was suspended from teaching and put on administrative leave by school . On monday she was arrested again for having sex with another pupil .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M1",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "New zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nNew zealander tim weston was on holiday in puerto vallarta with his wife when the attack happened and described how ` the crocodile kept the dog in its mouth for ages , not moving at all ' . The clip shows a large croc calmly moving through the marina in the tourist hotspot of puerto vallarta , in mexico , with a black dog wearing a red collar lifeless in its mouth . A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-b5925c1a02fa9a8f6b1d5ad764c19a732deac0df",
+ "system_id": "M22",
+ "source": "A video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '",
+ "reference": "A local bike shop owner 's dog was snatched by a crocodile . The crocodile was swimming through Puerto Vallarta marina , in Mexico . New Zealander Tim Weston was on holiday and saw the bizarre sight . A video of the dog in the croc 's jaws has been viewed half a million times .",
+ "system_output": "A video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA video of a menacing crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico , with a black dog wearing a red collar lifeless in its mouth . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when the attack happened and described how 'the crocodile kept the dog in its mouth for ages , not moving at all ' . Mr Weston told Daily Mail Australia the dog belonged to the local bike shop 's owner . Scroll down for video The large croc proudly held on to its catch for hours as it lurked around the Puerto Vallarta marina , in Mexico 'There 's a sanctuary for these crocodiles up the river ( 200m away ) which runs out to the sea and the marina , ' Mr Weston explained . 'The dog went to the water 's edge to see the crocodile ( being curious I guess ) but things did n't go according to plan obviously . 'It was certainly a shock to the system . Guess that 's nature though , ' he said . In the video the beast glides through the water , with rocks on one side and a footpath on the other , as crowds gasp in amazement . New Zealander Tim Weston was on holiday in Puerto Vallarta with his wife when he saw the alarming sight Tourist Tim Weston took a photo of the crocodile with the dog between its jaws A video of the crocodile swimming through the marina with the bike shop owner 's dog has been viewed more than 500,000 times Mr Weston said : ' A couple of hours later we walked back past and there was still a crowd ... But we did n't want to look at it again as we had just eaten . Yesterday morning there was no sign of either . ' Social media users have reacted in horror to the video that was posted on the Bogged Facebook page . One woman called the video 'horrible ' and another said ' I hate crocodiles now ' . But crocodiles are a common site in Puerto Vallarta as the Mexican beach resort is built amongst the mangroves that the creatures have called home for centuries . As well as living in the nearby sanctuary , crocs are regularly spotted on the resort 's golf course . In Mexico the 'American Crocodiles ' can reach lengths of 6 meters and as a result of urbanisation are living closer to humans than ever before . One social media user called the video 'horrible ' and another said ' I hate crocodiles now '\n\nSummary:\n\nA video of a crocodile swimming through a public marina with a dead pet dog locked in between its jaws has been viewed more than half a million times online . The clip shows a large croc calmly moving through the marina in the tourist hotspot of Puerto Vallarta , in Mexico . New Zealander Tim Weston was on holiday with his wife when the attack happened .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6721edf88c95f480a4db2c63649dbb03736aeadf",
+ "system_id": "M12",
+ "source": "A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .",
+ "reference": "Ian Guffick , 31 , allowed pupils to make changes outside exam conditions . He also made changes to the pupil 's work before the papers were sent off . Probe was launched after officials were tipped off he breached exam rules . A disciplinary panel has now banned Guffick from teaching for two years .",
+ "system_output": "Ian guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole school 's SATs results . Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess English , maths and science at Mitton Manor Primary School in Tewskesbury , Gloucestershire . This was despite the exam session finishing , a conduct panel heard . Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury , pictured , to make the changes to their SATs exam answers The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off . After the suspected breach of exam rules , school officials were tipped off , resulting in an investigation being launched in June . The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same . It then led to the Department for Education annulling all SATs exam results for the entire school last year , the hearing was told . Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct . The Department for Education annulled the whole school 's SAT 's results after a probe by the local education authority ( file picture ) However , he insisted in a statement that he did not ask the pupils to change their answers , but to make them more legible . But the panel , run by the National College for Teaching and Leadership , said that no changes of any kind should be made to exam scripts outside test conditions . Panel chair Martin Pilkington said , although the teacher \u2019 s conduct had been dishonest , it fell short of fraud or serious dishonesty . But he added : 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school , that can in no way excuse his behaviour . 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority . ' Paul Heathcote , acting on behalf of Education Secretary Nicky Morgan , banned Guffick from the classroom for at least two years . He said : 'His actions had a serious impact on the school as their SATs results were annulled for 2014 . 'Mr Guffick \u2019 s actions were deliberate and he was not acting under duress . 'In all the circumstances , I agree with the panel \u2019 s recommendation that a prohibition order is an appropriate and proportionate sanction . 'Mr Guffick has shown remorse and insight and he deeply regrets his actions . ' He was given 28 days in which to appeal against the decision to the High Court .\n\nSummary:\n\nIan guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds , which assess english , maths and science at mitton manor primary school in tewskesbury , gloucestershire . This was despite the exam session finishing , a conduct panel heard . The 31-year-old also altered a number of the pupil 's exam answers himself before the papers were sent off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-97bbda7ec22a736cd174b51513c72a6372d13d35",
+ "system_id": "M8",
+ "source": "The glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .",
+ "reference": "The Yankee Stadium signage was bought by Hall of Famer Reggie Jackson when the stadium was shuttered in 2008 for an undisclosed sum . Before the auction , Jackson had promised to fly out himself to see the letters at the winner 's home .",
+ "system_output": "13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThe glowing blue letters that once lit the Bronx from above Yankee Stadium failed to find a buyer at an auction at Sotheby 's on Wednesday . While the 13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 , the only person who raised a paddle - for $ 260,000 - was a Sotheby 's employee trying to jump start the bidding . The current owner of the signage is Yankee Hall of Famer Reggie Jackson , who purchased the 10-feet-tall letters for an undisclosed amount after the stadium saw its final game in 2008 . No love : 13 letters that hung over Yankee Stadium were estimated to bring in anywhere from $ 300,000 to $ 600,000 , but received no bids at a Sotheby 's auction Wednesday The 68-year-old Yankee said he wanted ' a new generation to own and enjoy this icon of the Yankees and of New York City . ' The letters had beamed from atop Yankee Stadium near Grand Concourse in the Bronx since 1976 , the year before Jackson joined the team . ESPN reports Jackson , who attended the auction , bought the letters 'on a lark . ' Jackson joked that once he saw the size of the letters , some of which are also 10-feet-wide , he realized he might have gotten ahead of himself . He promised before the auction that if the letters were bought by a bidder in the United States , he would fly out to see the signage himself at the winner 's home . Yankee Stadium : The letters at the stadium , which was built in 1923 , were installed in 1976 and taken down in 2008 In addition to the letters , Sotheby 's was auctioning off a New York Mets bullpen car from the late 60s . Despite the Mets memorabilia being estimated at only $ 20,000 to $ 30,000 , it raked in a winning bid of $ 112,500 at auction .\n\nSummary:\n\n13 letters were expected to bring in anywhere from $ 300,000 to $ 600,000 . The letters had beamed from atop yankee stadium near grand concourse in 1976 , the year before jackson joined the team . The 68-year-old yankee said he wanted ' a new generation to own and enjoy this icon of the yankees and new york city ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M1",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool manager brendan rodgers has reminded raheem sterling and jordon ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the england international who will lead liverpool 's line at wembley on sunday in the absence of the injured daniel sturridge , was exposed for inhaling the legal high nitrous oxide on monday . Then 24 hours later pictures emerged of sterling and ibe with shisha pipes .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-3ba3bde4a1440134bb38ec22ba50db59986b544c",
+ "system_id": "M20",
+ "source": "Liverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley",
+ "reference": "Liverpool face Aston Villa in their FA Cup semi-final at Wembley on Sunday . Liverpool forwards Raheem Sterling and Jordon Ibe were pictured with a shisha pipe earlier this season . Neither have been disciplined by Reds boss Brendan Rodgers for incident .",
+ "system_output": "Liverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 3.0,
+ "overall": 2.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nLiverpool manager Brendan Rodgers has reminded Raheem Sterling and Jordon Ibe about their professional responsibilities and urged them to learn from a chastening week . Sterling , the England international who will lead Liverpool \u2019 s line at Wembley on Sunday in the absence of the injured Daniel Sturridge , was exposed for inhaling the legal high nitrous oxide on Monday . Then 24 hours later pictures emerged of Sterling and Ibe with shisha pipes . Raheem Sterling ( right ) was pictured smoking a shisha pipe with team-mate Jordon Ibe earlier this season Liverpool boss Brendan Rodgers ( left ) has n't disciplined Sterling ( pictured ) or Ibe for the incident Ibe was present among the Reds squad during their training session on Friday ahead of their FA Cup semi Rodgers , who has Steven Gerrard available for the FA Cup semi-final with Aston Villa , spoke to Sterling and Ibe in his office on Thursday but neither will be disciplined and Rodgers now wants to draw a line under the matter . He said : \u2018 It is understanding that when you are a professional , elite sportsperson , there are things you can and can not be doing and be seen to be doing . \u2018 It \u2019 s not just for your welfare but also because of your responsibilities . \u2018 It is difficult . Some of these young guys are doing what you would consider to be normal things \u2014 and I \u2019 m not talking about this instance \u2014 ( but ) how it gets framed is totally different. \u2019 Rodgers would not discuss whether he would give Gerrard , available after a three-match suspension , an immediate recall against Villa , making it clear that his sole priority is steering Liverpool into their first FA Cup final of his reign . They fell short in the title race last season and then faltered in the last four of the Capital One Cup in January , but Rodgers thinks they are ready to progress . Reds have captain Steven Gerrard ( centre ) back for their FA Cup semi after serving a three-match suspension \u2018 When you get so close , whether it was in the title race or the cup , it hurts , \u2019 said Rodgers . \u2018 Trace back the winners in sport . If you look at everything they have won , also look at what they have lost . You will see that they go close on a number of occasions before they step on . \u2018 Look at players who have won Champions Leagues and you trace back and see they have also lost Champions League semi-finals and finals . It \u2019 s all part of the journey \u2014 losing but taking that forward and improving you. \u2019 Rodgers ( left ) has n't revealed whether he 'll recall Gerrard for Sunday 's clash against Aston Villa at Wembley\n\nSummary:\n\nLiverpool captain Steven Gerrard ( left ) was unavailable for selection for the FA Cup semi-final against Aston Villa 'There are just so many things you ca n't do . You ca n't be a professional and be a professional and smoke weed . ' I 'm not going to comment on things that are out in the public domain , but you have to understand what you can and ca n't do .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M14",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil was in fine form as germany beat georgia 2-0 in euro 2016 qualifier on sunday . Ozil impressed on international duty as germany beat gerogia 2-0 in a euro 2016 qualifier on sunday . Ozil was said to have been spotted in a berlin nightclub after missing the gunners ' 2-1 win at st james ' park .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-4e0de0ec6d5b9df174f54d03158de8f30aac861f",
+ "system_id": "M23",
+ "source": "Arsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year",
+ "reference": "Mesut Ozil starred as Germany beat Georgia 2-0 in Euro 2016 qualifier . Playmaker returns to London ahead of Arsenal 's game against Liverpool . Ozil puts his feet up on sofa with his dog as he enjoys home comforts .",
+ "system_output": "Mesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nArsenal playmaker Mesut Ozil seemed to be in a enjoying his home comforts after arriving back in London following the international break , posting a picture putting his feet up with his dog . The German star was in fine form as the World Cup winners beat Georgia 2-0 in their Euro 2016 qualifier on Sunday , and is now setting his sights on the Premier League encounter with Liverpool at the weekend . And Ozil gave an insight into what he does in his downtime after taking a snap of him playing with his pet with the caption : 'Home Sweet Home ' . Mesut Ozil posted a picture on Twitter relaxing with his dog on Wednesday after returning home to London Ozil impressed on international duty as Germany beat Gerogia 2-0 in a Euro 2016 qualifier on Sunday The midfielder , who was signed for \u00a342.5million from Real Madrid in 2013 , missed Arsenal 's last game against Newcastle through illness , but will be expected to start against Liverpool at the Emirates Stadium . Ozil was said to have been spotted in a Berlin nightclub after missing the Gunners ' 2-1 win at St James ' Park , though manager Arsene Wenger rubbished those claims . He has been doing the business on the pitch after returning from injury though , wracking up four assists and scoring three goals since January . And the former Schalke man says he is finally adjusting to English football , claiming he fits in 'perfectly ' . 'There are a lot of what we call 'English weeks ' in Germany - where you play twice , and there 's no winter break either , ' he said . 'It 's demanding because you do n't have many breaks to recover , but as a footballer it 's great to have so many games and I 'm the sort of player who prefers playing in many matches - that 's why I fit perfectly here . ' The playmaker has been in fine form since returning from injury for Arsenal this year\n\nSummary:\n\nMesut ozil posted a picture on twitter playing with his dog on wednesday . The arsenal playmaker returned from international duty on sunday . Ozil was in fine form as germany beat georgia 2-0 in a euro 2016 qualifier . He will be expected to start against liverpool at the weekend .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M11",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Jordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.6666666666666665,
+ "fluency": 2.3333333333333335,
+ "relevance": 3.0,
+ "overall": 2.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nJordan henderson is set to sign a new long-term contract at anfield . The club 's vice-captain had 14 months remaining on his current contract . Henderson is the third major player in liverpool 's fa cup . The fa cup fourth round . Raheem sterling is expected to return to liverpool in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-6f027d030f9ca86446dc893f1d10ced4c9a3561f",
+ "system_id": "M20",
+ "source": "Jordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer",
+ "reference": "Jordan Henderson is set to sign an improved deal with Liverpool . The 24-year-old midfielder has 14 months left on his current contract . Henderson could replace Steven Gerrard as club captain this summer . Liverpool will resume talks with Raheem Sterling at the end of the season .",
+ "system_output": "Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.3333333333333335,
+ "fluency": 1.6666666666666667,
+ "relevance": 3.0,
+ "overall": 2.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nJordan Henderson has provided Liverpool with a lift after their FA Cup heartache by agreeing a new long-term contract . The club 's vice-captain had 14 months remaining on his current contract and his advisors had been in talks with Liverpool since the beginning of this season . They have now reached a resolution and Henderson is expected to put pen-to-paper on improved terms that are likely be worth in the region of \u00a3100,000 . His new deal will run to 2020 . Liverpool midfielder Jordan Henderson is set to sign a new long-term contract at Anfield Henderson chases down Aston Villa 's Jack Grealish during Liverpool 's FA Cup semi-final defeat at Wembley Henderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson , 24 , is the third big player in Brendan Rodgers ' squad to agree a contract extension , following on from Daniel Sturridge and Philippe Coutinho . The England international , who was signed by Kenny Dalglish in June 2011 for \u00a316million from Sunderland , has been one of the most improved players under Rodgers ' watch . His form this season has been excellent and he has contributed 13 assists as well as seven goals from midfield ; he will be considered for the role of club captain when Steven Gerrard moves to LA Galaxy . Talks with Raheem Sterling are not expected to resume until the end of the season but Ian Ayre , Liverpool 's Chief Executive , last week said he expected the England forward to be at Anfield for ' a long time ' . Henderson could replace Steven Gerrard as Liverpool captain when the 34-year-old departs this summer Liverpool boss Brendan Rodgers ( right ) is keen to tie-down Henderson with up to 10 players set to leave Raheem Sterling has rejected a new deal at Liverpool but talks are expected to resume in the summer\n\nSummary:\n\nHenderson 's new deal is worth around \u00a3100,000-a-week and will run until the summer of 2020 Henderson is set to make a \u00a310million move to Manchester City this summer but is keen to stay at Anfield Henderson has been one of the most improved players under Brendan Rodgers ' watch Henderson has been one of Liverpool 's most improved players this season and has contributed 13 assists Henderson last week said he was 'rel\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M13",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "No less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nNo less than 13 out of 22 players on show at kick-off were english . Callum gribbin was the matchwinner for manchester united with a delightful free-kick . Altrincham 's compact 6,000-capacity home may not be old trafford , but it does have a proud history of its own .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-d89de9c2a76f2665560becfe5d761a4fc62b9926",
+ "system_id": "M20",
+ "source": "First blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .",
+ "reference": "Man United beat Man City 1-0 in Saturday 's under 18 encounter . Callum Gribbin scored the winning goal at Moss Lane , Altrincham . The result leaves United two points behind leaders Middlesbrough . The 'real ' Manchester derby takes place at Old Trafford on Sunday .",
+ "system_output": "United went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nFirst blood to United after their Under 18s saw off City 1-0 in the 'mini-derby ' . Kits aside , this was probably as far removed from Sunday 's big match as you could imagine . For a start , no less than 13 out of the 22 players on show at kick-off were English . Of those , 10 were Mancunian . Callum Gribbin was the matchwinner for Manchester United with a delightful free-kick Ticket prices ? Entry was free and close to 1,000 gathered on the seats and terraces of Moss Lane for a match that kicked off at 3pm on Saturday with half-and-half scarves nowhere to be seen . Altrincham 's compact 6,000-capacity home may not be Old Trafford , but it does have a proud history of its own . It was certainly a grander stage than a windswept Saturday morning on an outfield at United 's Carrington complex , where Paul McGuinness 's Under 18s usually ply their trade . The young Reds coach wanted to make the experience closer to what his tyros could expect should they make the step up to the seniors . And his side lined up with three at the back and supporting wingbacks in a formation seen more than once this season in the first team . In an even first-half the impressive Marcus Wood , from just down the road in Sale , came closest for City with an audacious chip . United manager Paul McGuinness saw his side claim victory in the 'mini derby ' For the home side towering centre-forward Marcus Rashford , another local lad from whom big things are expected , wasted two decent opportunities when put through . Just before the hour mark England Under 17 star Callum Gribbin made and scored the first . In March , the nifty-footed attacking midfielder went viral when United tweeted a clip of him showing outrageous skills to trick four Aston Villa players . He was at it again here , showing nifty footwork on the edge of the box and drawing a foul . After dusting himself down Gribbin stepped up to curl a delightful free-kick around the wall and beyond blonde Joe Hart-a-like Charlie Albinson in the City goal . Moments later it was almost two when full-back Cameron Borthwick-Jackson 's rasping 25-yard drive struck the angle of the crossbar and post . The same man looked to have gotten away with one when he appeared to clip Buckley in the box before sub Aaron Nemane hit the side-netting as the visitors looked for an equaliser . Dutch winger Javairo Dilrosun curled a free kick onto the roof of the net in stoppage time for City but that was all she wrote . The result leaves United , who have won two of this year 's cross-town clashes , two points behind leaders Middlesbrough with a game in hand . Injury-ravaged City however , have now lost both matches in the Barclays Under 18s Premier League end-of-season play-off series . FA chief Greg Dyke recently bemoaned the lack of first-team opportunities for homegrown talent between the ages of 18 and 21 . On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the On the evidence of this entertaining 90 minutes high on skill and commitment , there are talents on both sides that may well deserve their chance in the coming years .\n\nSummary:\n\nUnited went in front after a quick break , with Rashford heading home a free-kick from Gribbin . The young Reds coach 's team were at their best . And they were on top for the majority of the half .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b59fa7715418c4bcc0d50e3caeb239129de453e",
+ "system_id": "M11",
+ "source": "England ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps",
+ "reference": "Joe Hart has revealed he is inspired by the likes of Gianluigi Buffon . Italy 's record cap holder Buffon played his 147th game for the Azzurri . Man City shot stopper Hart recorded his 50th appearance for England .",
+ "system_output": "England star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 1.6666666666666667,
+ "fluency": 1.0,
+ "relevance": 1.6666666666666667,
+ "overall": 1.4166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nEngland ace Joe Hart labelled fellow goalkeeper Gianluigi Buffon a 'legend of the game ' after seeing the Italian veteran claim his 147th cap . Hart , who passed an impressive milestone of his own by representing his country for the 50th time on Tuesday night , said after the 1-1 draw with Italy that Buffon was an inspiration . Speaking to FA TV , Hart said : ' I \u2019 m still learning my game and I \u2019 m still watching the likes of Buffon and the way he goes about his business at 37 years old . England and Manchester City goalkeeper Joe Hart has lavished praise on fellow goalkeeper Gianluigi Buffon Hart has labelled Buffon , who won his 147th cap against England on Tuesday night , a 'legend of the game ' ' I \u2019 ve got a lot more learning to do and I want to do it in this team . ' [ Buffon \u2019 s caps total ] is a long way off , but it \u2019 s definitely a night to celebrate a terrific goalkeeper and a legend of the game , someone I personally look up to and it \u2019 s inspirational to see . ' The Manchester City shot stopper , who is 10 years younger than Buffon , revealed his delight at receiving his 50th cap at the Juventus Stadium . ' I was proud of my first cap , I was proud to represent the Under 21s , and 50 caps at my age is good , ' added Hart . ' I want to keep going , that \u2019 s not the end for me . I just want to keep going , keep playing well for my club and country and rack them up . ' England ace Hart , pictured saving a shot by Citadin Eder , is desperate to add to his 50 international caps\n\nSummary:\n\nEngland star joe hart labelled the ` legend of the game ' . The 1-1 draw with italy on tuesday night . Gianluigi buffon has labelled buffon , who won his 147th cap against england .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M17",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders in 67 cities . The app hires independent contractors who drive their own cars in a similar model to uber taxis . Delivery fees for postmates begin at $ 5 and up , depending on distance and demand .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-1747dcd6a007ee97954b6b10cde549bf82f6a4eb",
+ "system_id": "M9",
+ "source": "Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle",
+ "reference": "Mexican restaurant has decided to tap into $ 70 billion food delivery market . Fast-casual chain will work with the Postmates app to allow mobile orders . App works in similar way to Uber , using hired drivers to deliver the food . But the chain will add a 9 % service charge - on top of Postmates ' $ 5 rate .",
+ "system_output": "The fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.083333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nMexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors . The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities , according to a report in Nation 's Restaurant News . But Mexican food fans should know that the restaurant plans to add a nine per cent service charge - with the delivery fees for Postmates beginning at $ 5 and up , depending on distance and demand . New venture : Mexican restaurant Chipotle has decided to tap into the $ 70 billion food delivery market by teaming up with an app to bring burritos straight to customers ' doors Chipotle is thought to have turned down previous offers of collaboration with third-party delivery services - but has now given Postmates the green light , New York Daily News reports . The app hires independent contractors who drive their own cars in a similar model to Uber taxis . Chipotle has gone from strength to strength in recent years and this month two presidential candidates have been spotted in their restaurants . Republican contender Marco Rubio is a regular in the outlets and recently told MailOnline that he always tips staff . Cashing on delivery market : The fast-casual chain will work with the Postmates app to begin offering delivery for online and mobile orders in 67 cities And just one day into her presidential campaign , Hillary Clinton was seen in a Chipotle in Maumee , Ohio where she ordered a chicken dish with guacamole , a chicken salad and fruit juice . Chipotle is not the only franchise hoping to cash in on the lucrative delivery market . Starbucks , which will deliver cups of coffee in parts of Seattle and New York by the end of the year , has also partnered with Postmates . Taco Bell recently announced plans to test a delivery service this year , McDonald 's 'McDelivery ' program is in the planning process and Burger King has launched a pilot program in select markets , including parts of New York City , through bkdelivers.com . Celebrity customers : Hillary Clinton was spotted ordering a meal at Chipotle in Ohio earlier this month - one day into her presidential campaign Another candidate : Security camera footage obtained by conservative news publication IJ Review released online this month shows Florida senator Marco Rubio visiting a Washington , D.C. Chipotle\n\nSummary:\n\nThe fast-casual chain will work with the postmates app to begin offering delivery for online and mobile orders . Mexican food fans should know the restaurant plans to add a nine per cent service charge . Chipotle has decided to tap into $ 70 billion food delivery market . Chipotle is thought to have turned down previous offers of third-party delivery services . Mexican restaurant chipotle has decided to tap into the $ 70 billion food delivery .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M15",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "The sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 3.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\nThe sentencing phase begins tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . Bill and denise richard wrote about the toll taken on their family after the death of their 8-year-old son , martin .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-e49792c337d3f4c13e22f710efa44cf6a4e59aba",
+ "system_id": "M0",
+ "source": "( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .",
+ "reference": "Parents of Martin Richard argue against death penalty for Dzhokhar Tsarnaev . The 8-year-old boy was youngest of victims in the Boston Marathon bombings . Sentencing phase for Tsarnaev begins next week .",
+ "system_output": "( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.666666666666667,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) The parents of the youngest victim of the Boston Marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , Dzhokhar Tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges The sentencing phase begins Tuesday , a day after this year 's edition of the landmark race . It is expected to last four weeks . The 13th Juror : Now it gets real In a front-page opinion piece in The Boston Globe , Bill and Denise Richard wrote about the toll taken on their family after the death of their 8-year-old son , Martin . Their daughter , Jane , also was severely injured . `` Our family has grieved , buried our young son , battled injuries , and endured numerous surgeries -- all while trying to rebuild lives that will never be the same , '' they said in the Globe column titled `` To end the anguish , drop the death penalty . '' `` We sat in the courtroom , day after day , bearing witness to overwhelming evidence that included graphic video and photographs , replicated bombs , and even the clothes our son wore his last day alive . '' They said they understood the `` heinousness and brutality of the crimes committed . '' `` We were there . We lived it . The defendant murdered our 8-year-old son , maimed our 7-year-old daughter , and stole part of our soul . '' But now the Richards are urging the Justice Department to bring the case to a close . `` We are in favor of and would support the Department of Justice in taking the death penalty off the table in exchange for the defendant spending the rest of his life in prison without any possibility of release and waiving all of his rights to appeal , '' they wrote . They go on to say : `` We know that the government has its reasons for seeking the death penalty , but the continued pursuit of that punishment could bring years of appeals and prolong reliving the most painful day of our lives . We hope our two remaining children do not have to grow up with the lingering , painful reminder of what the defendant took from them , which years of appeals would undoubtedly bring . '' Martin Richard and two others were killed and more 200 people wounded when a pair of bombs went off within 12 seconds of each other at the finish line on April 15 , 2013 . Tsarnaev was convicted last week , while his brother , Tamerlan , was killed in a shootout with police two years ago . The Richards never mention Tsarnaev by name . They stress that they were only speaking for themselves when they argue against the death penalty . `` We believe that now is the time to turn the page , end the anguish , and look toward a better future -- for us , for Boston , and for the country , '' they wrote .\n\nSummary:\n\n( cnn ) the parents of the youngest victim of the boston marathon bombings are making an emotional , passionate plea to take the death penalty off the table for the man convicted in the case . Last week , dzhokhar tsarnaev was found guilty on all 30 charges he faced related to the bombings at the 2013 race and the dramatic violence that dragged out for days afterward . A look at all of the charges .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d296270ab4a4cf20f2d9c1aae7514687806f2b35",
+ "system_id": "M5",
+ "source": "These extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers",
+ "reference": "Boeing 747 can normally hold up to 600 people but this model was customised for a single ultra-wealthy tycoon . Digital images show how the interior is as luxurious as an expensive hotel room despite space constraints . The jumbo jet contains a master bedroom , 'aeroloft ' with added sleeping space , and a large dining room . It is also kitted out for business with a conference room and office so the owner is never out of touch with work .",
+ "system_output": "The custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThese extraordinary images show how an enormous jumbo jet has been converted into a luxury home for a mystery billionaire - complete with bedrooms , multiple lounges and an onboard restaurant . The custom-built Boeing 747 is believed to have cost its ultra-wealthy owner \u00a3400million after it was customised to fit his specifications exactly over a three-year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and 'state room ' at the back of the plane . More sleeping space is provided in the 'aeroloft ' on the top deck of the vessel , with eight full beds for passengers who prefer to get some shut-eye on flights . Luxury : This is the lounge on board a custom-built Boeing 747-8 , as depicted in an artist 's impression by Greenpoint Technologies Shuteye : The master bedroom of the \u00a3400million plane has a huge bed as well as an armchair and sofa for when the owner needs to relax Business : The conference room of the plane could be perfect for meetings in the sky with its long table in the centre Chill-out : The lounge decor looks like a luxury hotel , belying its true location inside a plane similar to those used by commercial airlines Executive : This room , which can be used as an office or a guest bedroom , gives a hint to its true location thanks to the porthole-style windows Kicking back : The main suite could easily be mistaken for a room in a top hotel thanks to its over-the-top decorations Restaurant : The central portion of the plane hosts a dining room which is attached to a kitchen , allowing the owner to make use of the services of a private chef Shiny : The jumbo jet is one of the most luxurious private planes ever built , with the price tag believed to have reached a total of \u00a3400million Sofa : Parts of the plane are set aside for guests to relax while watching television , as the vessel can carry several dozen passengers Plush : Another artist 's impression of the main state room , which also contains a desk so the owner can switch between work and relaxation Welcoming : Images of the lounge show a space very different to the normal aeroplane cabin , with the traditional rows of seats completely discarded in favour of a hotel-like ambience Exclusive : Few people can ever imagine flying in this sort of luxury - far more glitzy than first class on commercial flights Plan : The layout of the Boeing 747 is shown in this plan published by Greenpoint Technologies Gigantic : The luxury interior has been built inside a Boeing 747-8 similar to this one , which can hold up to 600 passengers\n\nSummary:\n\nThe custom - built boeing 747 is believed to have cost its ultra-wealthy owner \u00a3 400million after it was customised to fit his specifications exactly over a three - year period . The jumbo jet would normally carry as many as 600 passengers - but this version was built for just a single tycoon , although he will be able to fit dozens of guests . It contains a large dining room where travellers can enjoy meals cooked in an adjoining kitchen , with a lounge and ' state room ' at the back of the plane .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M17",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk says the rocket landed , but tipped over . Spacex is trying to land rockets at sea and on land to cut costs . This is the sixth spacex mission to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M23",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "Spacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\nSpacex founder elon musk tweeted that the rocket landed , but tipped over . The rocket is carrying an uncrewed cargo spacecraft called dragon to the international space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-b8b6e729fff27c4eef87887e61d3448de9c063f6",
+ "system_id": "M2",
+ "source": "( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .",
+ "reference": "SpaceX founder Elon Musk : `` Rocket landed on droneship , but too hard for survival . '' This was the second attempt at historic rocket booster barge landing . Dragon spacecraft will head toward International Space Station on resupply mission .",
+ "system_output": "( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .",
+ "scores": {
+ "coherence": 1.0,
+ "consistency": 4.666666666666667,
+ "fluency": 3.3333333333333335,
+ "relevance": 2.3333333333333335,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) SpaceX on Tuesday launched a two-stage Falcon 9 rocket carrying an uncrewed cargo spacecraft called Dragon on a flight from Cape Canaveral , Florida , to the International Space Station . That was the easy part . In a difficult bid to land a rocket stage on a floating barge for the first time , the private space exploration company was unsuccessful . SpaceX founder Elon Musk tweeted : `` Ascent successful . Dragon enroute to Space Station . Rocket landed on droneship , but too hard for survival . '' He later clarified that the rocket landed , but tipped over . SpaceX tried to land a Falcon 9 on the drone ship in January , but the rocket hit at an angle and exploded . SpaceX has said it will keep trying and , after it masters landing at sea , hopes to someday land rockets on the ground . Usually booster rockets burn up in Earth 's atmosphere or , like NASA 's space shuttle boosters , they fall back into the ocean . So why try to land one ? Musk wants to cut costs . On his company 's website , he says that if anyone can figure out how to `` reuse rockets just like airplanes , the cost of access to space will be reduced by as much as a factor of a hundred . '' What about the rest of the rocket and the Dragon ? The smaller , top part of the rocket will carry the Dragon into orbit and then break away from the cargo ship and burn up in Earth 's atmosphere . The Dragon will dock with the space station a couple of days after launch to deliver more than 4,300 pounds ( 1,950 kilograms ) of supplies , including research equipment and ISSpresso , an espresso maker that astronauts can use to make coffee and tea . The space station crew will spend about five weeks unpacking the Dragon . They 'll then stuff it with over 3,000 pounds of science experiments , trash and other stuff to send back to Earth . When they 're done , Dragon will leave the space station and mission controllers will guide it to splashdown in the Pacific Ocean off California . This is the sixth SpaceX mission to the International Space Station . The company was the first private space contractor to dock with the station . Tuesday 's launch was the second attempt for this mission . Monday 's planned launch was scrubbed due to weather . CNN 's Catherine E. Shoichet contributed to this report .\n\nSummary:\n\n( cnn ) spacex on tuesday launched a two-stage falcon 9 rocket carrying an uncrewed cargo spacecraft called dragon on a flight from cape canaveral , florida , to the international space station . That was the easy part . Dragon enroute to space station .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2feca9acf532e33f1ab7442c442c8e19787d8d7b",
+ "system_id": "M0",
+ "source": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .",
+ "reference": "Roland Giroux captured his relationship with his fish on camera . The blood parrot cichlid swims in circles and waits in man 's hand . The man then begins massaging and stroking its body with his finger . Mr Giroux claims the fish becomes frustrated if not stroked each day .",
+ "system_output": "Cats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that doesn \u2019 t prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , Roland Giroux gently cups the fish with his hand from within side the tank and then treats it to a massage . The fish begins relaxing in the man 's hand and stays very still as the owner begins to stroke it with his finger The fish initially moves away before swimming in a full circle and returning to rest in between the man \u2019 s fingers . The man then begins massaging the pet by running his finger along it and the fish stays completely still as if enjoying the sensation . Repeating the process , the fish swims off once more before immediately returning to the man \u2019 s hand to be stroked again . Later in the video the fish interacts further with the man by swimming up his arm and circling his fingers . The man places his hand into the tank and gently cups the blood parrot cichlid with his fingers Before burying its face into his hand and remaining very still once more . The video concludes with the man removing his hand from the tank and the fish , clearly saddened by this , following it up to the surface of the water . According to Mr Giroux , who uploads many videos of his fish to his Youtube channel , the blood parrot cichlid used to share its tank with two other fish . The fish swims away from the man 's hand a few times in the video but returns to be massaged But he was forced to remove the other two from the tank when he noticed that they were bullying his prized pet . He then set about interacting with the fish by feeding it frozen blood worms by hand and eventually he noticed it becoming more sociable . After a while the fish enjoyed being stroked and Mr Giroux noted that it would become frustrated if he didn \u2019 t play with it on a daily basis . The fish begins swimming around the man 's fingers and towards the surface as he removes his hand The man massages and plays with his fish for at least 15 minutes each day . The blood parrot cichlid is a hybrid of the midas and the redhead cichlid and was first bred in Taiwan around 1986 .\n\nSummary:\n\nCats and dogs have the advantage over marine pets in that they can interact with humans through the sense of touch . But that does n't prevent one man from dipping his hand in the fish tank and giving his blood parrot cichlid a stroke . Capturing his remarkable relationship on camera , roland giroux gently cups the fish with his hand from within side the tank and then treats it to a massage .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M11",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.6666666666666667,
+ "fluency": 4.0,
+ "relevance": 2.6666666666666665,
+ "overall": 2.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in england cricket team . Stokes broke his neck during the ashes series of ups and downs . He has been dismissed last year and that will be a reminder to him . He is back in england 's kensington oval this week .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8b51a10370a1219137e231192e65b9cd52238ea6",
+ "system_id": "M13",
+ "source": "There would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .",
+ "reference": "Ben Stokes returns to the ground where he broke his wrist hitting a locker . He clashed with West Indies batsman Marlon Samuel in the second Test . But his aggression , when controlled , is important to England .",
+ "system_output": "Ben stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThere would have been no mercy in the dressing room . Nothing is sacred in the team . When Ben Stokes first turned up at the Kensington Oval this week someone would have said : \u2018 Mind that locker , Stokesey. \u2019 It was here , of course , that Stokes broke his wrist punching a locker after being dismissed last year and that will be a reminder to him of the need to channel his aggression in the right way now he is back in the England team . Ben Stokes has forged a reputation as a fiery character in the England cricket team Stokes ( left ) clashed with Marlon Samuels during the second Test between West Indies and England But Stokes ( right ) responded positively to the row after talking to England coach Peter Moores Only a slight niggle in his back might stop Stokes playing here in Barbados and I am glad he has fought his way back after what has been a year of ups and downs . I like Stokes and I like his character at a time when the game can be perceived as producing robots and players who can not capture the imagination . He has something about him , as he showed when he was in Mitchell Johnson \u2019 s face during the Ashes series . He was not worried about the bombardment he would get in return . But I do believe he will need careful man-managing . Peter Moores , England \u2019 s coach , dragged him to one side after his clash with Marlon Samuels in the second Test and had a long chat with him and if it was to talk about how he should respond then the advice he gave was clearly right . The way he approached the Jamaican after that third-day salute was spot on . England must make sure they do not quash that natural exuberance and desire to take someone on . It will not always come off but we have to accept that there will be lows as well as highs with this lad . What Stokes must not do is try to live up to his reputation and become pigeon-holed . He can be feisty and fiery as long as it \u2019 s natural but his aggression has to be controlled . You want the character as long as the end product is runs and wickets . Stokes ( second left ) can use his aggression to bring runs and wickets to the England team Stokes ( left ) congratulates England captain Alastair Cook after the side won the second Test England want to have Stokes in the team as much as possible to help him hit his full potential Let him go a bit but rein him in occasionally . There is an argument here to play the second spinner in Adil Rashid and if England went down that route in Bridgetown the place of Stokes or Chris Jordan would come into question but they clearly want both to play as much as possible . This summer there will rarely be the need for two spinners so England want these two exciting cricketers to grow into the international game . They can only get better and that \u2019 s an exciting prospect .\n\nSummary:\n\nBen stokes has forged a reputation as a fiery character in the england cricket team . Stokes clashed with marlon samuels during the second test between west indies and england . But stokes responded positively to the row after talking to england coach peter moores .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f5fead94ee884800e84a212cc0edc78b11c4ba9f",
+ "system_id": "M17",
+ "source": "Manchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis",
+ "reference": "Evangelos Patoulidis also attracted interest from Barcelona and Arsenal . Anderlecht rejected a move to Barcelona when he was 12 . City in talks with Anderlecht chief Roger Vanden Stock to complete a deal .",
+ "system_output": "Evangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nManchester City are keen to sign Anderlecht teenager Evangelos Patoulidis . The 14-year-old playmaker is regarded as one of the best talents to emerge from Anderlecht 's youth set-up and has also attracted attention from Arsenal and Barcelona . The Belgian starlet rejected a move to Barcelona 's La Masia academy when he was 12 as his family wanted him to continue his studies . He has continued to impress and City have held discussions with Anderlecht chairman Roger Vanden Stock in the hope of agreeing a compensation package . Manuel Pellegrini is looked to build for the future by snapping up hot property Evangelos Patoulidis\n\nSummary:\n\nEvangelos patoulidis is regarded as one of the best talents to emerge from anderlecht 's youth set-up . The 14-year-old playmaker has also attracted attention from arsenal and barcelona . City have held discussions with anderlecht chairman roger vanden stock in the hope of agreeing a compensation package .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M11",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 5.0,
+ "relevance": 2.3333333333333335,
+ "overall": 2.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa santillan , 33 , was found in a flat in fulham , south west london . The 33-year-old mexican national was found dead at the end of march . She was found guilty of injuries to the head and neck . Police are appealing to anyone who may have had contact with miss santillan in the hours before her death . She has been told to return to a london police station for further questioning .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-36352fd1c61fd0c593ee7b84cbe213aa9b444439",
+ "system_id": "M22",
+ "source": "A man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .",
+ "reference": "Vanessa Santillan was found dead at a flat in Fulham , south west London . The 33-year-old Mexican national was working as a transgender escort . A 23-year-old man was arrested in connection with her death last month . He was bailed pending further inquiries as police continue investigation .",
+ "system_output": "Vanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA man has been arrested in connection with the death of a transgender escort who was found strangled and beaten last month . Vanessa Santillan 's body was found in a \u00a3400,000 flat in Fulham , south west London , at the end of March . The 33-year-old Mexican national , who worked as a transgender escort , died as a result of injuries to the head and neck . Vanessa Santillan , 33 , was killed at the end of March . Her body was found in a flat in Fulham having been strangled and beaten to death A 23-year-old man was arrested in connection with her death but has been bailed . He has been told to return to a London police station for further questioning at a later date . Meanwhile Scotland Yard is appealing to anyone who may have had contact with Miss Santillan in the hours before her death . According to her website Miss Santillan worked in London , Paris and Miami as an escort . Police did not confirm whether her profession was central to the investigation but insisted they would do 'everything ' to solve the case . London Ambulance Service was called to a flat in Romily Court , Fulham , on March 28 at around 9.30pm . Miss Santillan was pronounced dead at the scene having suffered injuries to her head and neck . The woman had been working as a transgender escort , her website revealed . Miss Santillan is understood to have moved to London from Mexico The woman , who described herself as visiting London from Miami , was pronounced dead at the scene last month Miss Santillan spoke of her love for London and Paris online in the weeks before her death . Police are urging anyone who had contact with her in the hours before her death to come forward Detective Chief Inspector Rebecca Reeves , who leads the investigation , said : 'We want to speak to anyone who saw Vanessa on Friday or Saturday . 'We need to know why this has happened and we want help from anyone who knew her while she was in London . ' In the weeks before her death Miss Santillan took to social media to talk of her love for London . On her website she described herself as visiting from Miami in search of 'upscale ' gentlemen . Miss Santillan 's body was found when London Ambulance Service was called to an address in Fulham , south west London Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nVanessa Santillan 's body was found in a \u00a3 400,000 flat in Fulham , London . The 33-year-old Mexican national died as a result of injuries to the head and neck . Police have arrested a man in connection with her death but have bailed him .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-a3b6201aca77b5bdaa5df02d6acee2be66233237",
+ "system_id": "M1",
+ "source": "A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show",
+ "reference": "The woman , named Dawn , was appearing on ITV 's The Jeremy Kyle Show . Accused her close friend and house-mate Jamie of stealing \u00a3207 . The money had been set aside for a grave stone for baby Daniel James . A sobbing Dawn revealed that the baby was the seventh she has lost .",
+ "system_output": "The bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone . The bereaved mother , named Dawn , was appearing on The Jeremy Kyle show and accused long-term chum , convicted burglar Jamie , of making off with the cash . When one of Mr Kyle 's famous lie detector tests revealed that Jamie , who lives with Dawn and her partner who is also called Jamie , had done the deed , she burst into tears and stormed off . Scroll down for video Devastated : Dawn , who has had seven miscarriages , was furious to discover her friend had stolen from her The drama began when Dawn , who has previously appeared on the ITV talk show , discovered that \u00a3207 set aside for a \u00a3500 grave stone had disappeared . Suspecting house-mate Jamie and his girlfriend Carol , she confronted them but was initially met with a flat denial . 'Only him and Carol knew where it was , ' she said during an angry confrontation on the ITV show . 'Me and Jamie had been saving up and only Carol knew where it was . ' Unable to prove it , she asked Mr Kyle to conduct a lie detector test and when the results came back , it showed that Jamie had indeed taken the cash . Caught out : Jamie , who has been to prison for burglary , was found to be lying after taking a test Outraged : Dawn , who was going to spend the money on a gravestone , was left furious by the revelation Mr Kyle also revealed that Jamie had sent a Facebook message to Dawn during the journey to London in which he declared his undying love and confessed to taking the money but blamed it on girlfriend Carol . Speaking after the show , a sobbing Dawn said : ' I 've had seven miscarriages and only one of them I 've had to bury . 'That was Daniel James and the \u00a3207 was put aside because the head stone was going to cost \u00a3500 altogether . 'We had to stand there and say goodbye to Daniel James and the last thing I wanted to do for Daniel James was put a head stone on him so he knew how much his mummy and daddy loved him . I feel betrayed , horrible . You try and do a good gesture to people and then you get it thrown in your face . ' Tears : She then burst into tears and rushed off the stage - pursued by the Jeremy Kyle cameras Sobbing : She was comforted by her partner , also called Jamie , after the show\n\nSummary:\n\nThe bereaved mother , named dawn , was appearing on the jeremy kyle show and accused long-term chum , convicted burglar jamie , of making off with the cash . When one of mr kyle 's famous lie detector tests revealed that jamie , who lives with dawn and her partner who is also called jamie , had done the deed , she burst into tears and stormed off . A woman who suffered seven miscarriages has spoken of her distress after a family friend stole the money set aside for her babies ' grave stone .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-e06ff0eb7c4fa4416fda84a9a239b15480c5a9d9",
+ "system_id": "M2",
+ "source": "A lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .",
+ "reference": "Man posted advertisement seeking 'girlfriend/wife ' on Gumtree . Rich , 31 , says potential partner would be showered with gifts and benefits . Post says the lucky girl would a gym membership , phone , laptop and car . 'Previous applicants need NOT apply ( NO exs ) ' the advertisement says .",
+ "system_output": "The outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA lonely man has taken his search for love - or lust - online , posting an advertisement complete with his desires , or 'requirements ' . The outback Casanova , who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree . 'WANTED ! ! ! ! ! girlfriend/wife , must love pets ... Accomodation available , position MAY offer company car , phone and laptop , to the 'successful candidate ' , ' it reads . Scroll down for video 'Rich' who lives in Humpty Doo , Northern Territory , detailed his dream girl on Gumtree 'Prefer NON drinker and smoker ( i know i 'm dreamin ) , fit attractive , will get free gym membership , to a gym of there choice , phone of there choice , must be reliable , honest , and loyal , good around th house , be good with their hands , good cook , great in bed . ' The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' . For his part , the 31-year-old only identified as Rich , described himself as : 'athletic build , white , heavily tattoo 'd brown hair , brown eyes , do n't drink , do n't smoke , DO N'T do drugs , like travelling , love the outdoors , sports , fishing , camping blah blah blah . ' The post continues : '175cm tall , WELL Endowed , own my own property ( s ) make great money , succesful , love cars , bikes , very independant . ' The 31-year-old 's advertisement included a list of requirements in his potential partner , and personal details about himself The advertisement outlined how the successful applicant must enjoy the outdoors , but is also 'also a princess when need to be ' Rich also said he is 'not shy of blowin cash on my woman ' , and his potential partner would 'receive lots of benefits ' . And , there 's no reason to be shy ladies , 'Rich ' promised 'all reasonable offers will be considered ' . The hopeless romantic also stated former partners need not apply . Daily Mail Australia unsuccessfully attempted to contact the man behind the commercial .\n\nSummary:\n\nThe outback casanova , who lives in humpty doo , northern territory , detailed his dream girl on gumtree . ` wanted ! ! ! ! ! Girlfriend/wife , must love pets ... accomodation available , position may offer company car , phone and laptop , to the ` successful candidate ' , ' it reads . ` rich ' who lives in humpty doo , northern territory , detailed his dream girl on gumtree .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M12",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "North pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nNorth pacific gray whale has earned a spot in the record books . The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-4761dc6d8bdf56b9ada97104113dd1bcf4aed3f1",
+ "system_id": "M2",
+ "source": "( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season",
+ "reference": "The whale , Varvara , swam a round trip from Russia to Mexico , nearly 14,000 miles . The previous record was set by a humpback whale that migrated more than 10,000 miles .",
+ "system_output": "The whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) A North Pacific gray whale has earned a spot in the record books after completing the longest migration of a mammal ever recorded . The whale , named Varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from Oregon State University , whose scientists helped conduct the whale-tracking study . Varvara , which is Russian for `` Barbara , '' left her primary feeding ground off Russia 's Sakhalin Island to cross the Pacific Ocean and down the West Coast of the United States to Baja , Mexico . Varvara 's journey surpassed a record listed on the Guinness Worlds Records website . It said the previous record was set by a humpback whale that swam a mere 10,190-mile round trip between the `` warm breeding waters near the equator and the colder food-rich waters of the Arctic and Antarctic regions . '' Records are nice , but Bruce Mate , the lead author of the study , thinks the long trip might say more about the whale than just its ability to swim . During her 14,000-mile journey , Varvara visited `` three major breeding areas for eastern gray whales , '' which was a surprise to Mate , who is also the director of the Marine Mammal Institute at Oregon State University . `` For her to go to Mexico , '' Mate said , `` It 's pretty strong evidence that it 's where she 's from . '' Varvara was thought to be an endangered western whale , but her ability to `` navigate across open water over tremendously long distances is impressive , '' he said in the release , which could mean that some western gray whales are actually eastern grays . With only 150 western gray whales believed to be in existence , that number might be even lower . `` Past studies have indicated genetic differentiation between the species , but this suggests we may need to take a closer look , '' Mate said . Fourth baby orca born this season\n\nSummary:\n\nThe whale , named varvara , swam nearly 14,000 miles ( 22,500 kilometers ) , according to a release from oregon state university , whose scientists helped conduct the whale-tracking study . Varvara , which is russian for `` barbara , `` left her primary feeding ground off russia 's sakhalin island to cross the pacific ocean and down the west coast of the united states to baja , mexico . Fourth baby orca born this season .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-8847383b72167c3f3dd2161b5b091b580c8f4da7",
+ "system_id": "M22",
+ "source": "Residents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass",
+ "reference": "A 1,000ft stretch of land rose up above sea level on Japanese island . The 100ft wide mass on Hokkaido is now some 30 to 50ft above sea level . It is believed to have been forced to the surface by a landslide nearby .",
+ "system_output": "1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nResidents on the east coast of Japan 's Hokkaido Island had an unexpected change of scenery this morning after 1,000ft of seabed was forced to the surface overnight . The extra stretch of coastline on Shiretoko Peninsula near the town of Rausu has risen as high as 50ft from the sea surface in some places , exposing what used to be the ocean floor . Geologists believe the emergence is a result of a landslide nearby , when melting ice and snow caused a section of land to drop , pivoting the underwater area into the air . Scroll down for video Surprise ! A man stands on a clump of land that emerged overnight along the coastline of Shiretoko Peninsula near Rausu , on Hokkaido Island , Japan 'An aerial survey indicates that the land swell happened as a result of a landslide ' , local researcher Yoshinori Yajima told the Hokkaido Shimbun newspaper after he flew over the area Monday . The mass , which has risen some 30 to 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide , a town official said . The new stretch of land came as a complete surprise to the locals , as there was no indication of the land movement overnight 'The local residents said they did n't hear any sounds and there were no tremors ( when the land appeared ) , ' said Katsuhiro Tanaka , the president of the Rausu Fisheries Cooperative Association , who viewed the expanded coastline the day it was discovered . The mass , which has risen some 50ft above sea level , measures roughly 1,000- 1,640ft long , and 100ft wide No clue : Local residents on Hokkaido island say they felt no tremors overnight , or heard any sounds to indicate movement , and that the landmass just appeared New view : Local residents from Rausu , Hokkaido , take a look at the newly emerged coastline Authorities have not been able to determine exactly when the landslide hit the snow-covered coastline , but they do not expect it to expand further , said an official at the central government 's Hokkaido Regional Development Bureau . 'Our understanding is that this is different from earthquakes , ' he told AFP . It is not believed to be related to the 7.8magnitude earthquake that struck the Kathmandu Valley on Saturday , which so far has claimed more than 4,000 lives . Under the sea : Marine organisms such as seaweed and sea urchins are still attached to the land mass\n\nSummary:\n\n1,000 ft of seabed was forced to the surface on Shiretoko Peninsula near the town of Rausu . Geologists believe the emergence is a result of a landslide nearby . melting ice and snow caused a section of land to drop , pivoting the underwater area into the air .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-801763443e6c73e3857039f2223798b2b58bef19",
+ "system_id": "M2",
+ "source": "Gregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold",
+ "reference": "Gregory van der Wiel does n't want to leave PSG in the summer window . The Dutchman revealed his 'love ' for the club and his team-mates . PSG continue to be linked with a move for Barcelona full back Dani Alves .",
+ "system_output": "Gregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nGregory van der Wiel has denied claims that he is set to leave Paris Saint-Germain in the summer and instead reiterated his 'love ' for the club . The Dutchman recently had a falling out with his boss Laurent Blanc after the decision to drop the full back for the Champions League clash against Chelsea . But Van der Wiel insists he 's happy in Paris and is making no plans to leave the Ligue 1 side . Gregory van der Wiel ( right ) says he 's not interested in leaving Paris Saint-Germain in the summer The full back had a recent spat with the manager Laurent Blanc ( right ) after he was dropped He told L'Equipe : ' I am genuinely happy at PSG . ' I love the club , I love the city and I love my team-mates . Everything is perfect for me here . 'There is absolutely no reason for me to leave . I want to sign a new contract . 'If the coach decides not to play me , that is his decision . But I am ready whenever I am called upon . ' The Parisians continue to be linked with Barcelona star Dani Alves , so it remains to be seen whether Van der Wiel has any choice in staying with the club . Dani Alves ( left ) is being linked with a move to the French champions and Van der Wiel could be sold\n\nSummary:\n\nGregory van der wiel has denied claims that he is set to leave paris saint-germain in the summer and instead reiterated his ` love ' for the club . The dutchman recently had a falling out with his boss laurent blanc after the decision to drop the full back for the champions league clash against chelsea . But van der wiel insists he 's happy in paris and is making no plans to leave the ligue 1 side .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M15",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c16c9fa34ccae7e2b9e29c3988ccb74974c4b3c0",
+ "system_id": "M8",
+ "source": "Carlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '",
+ "reference": "Carlos Tevez has reportedly told Juventus he wants to return to Argentina . Former Manchester City star wants to play for former club Boca Juniors . Club president Daniel Angelici urges striker to cancel his contract first .",
+ "system_output": "Carlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nCarlos Tevez has been told to terminate his contract with Juventus to complete a return to his former club Boca Juniors in Argentina . The former Manchester City striker 's deal with the Serie A champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Boca have confirmed they are close to completing a deal for the 31-year-old , but club president Daniel Angelici has stressed that Tevez must terminate his contract with the Italians first . Carlos Tevez has shocked Juventus by suggesting he wants to leave the club this summer Tevez is on course to win a second Serie A title with the Old Lady and still has a shot at European glory 'We must be careful ' , Angelici told TYC Sports . 'We know that he wants to return to Argentina with Boca Juniors but he must first terminate his contract with Juventus , which runs until 2016 . 'We are close to sealing his return and it will be done shortly . ' The Argentina international joined Boca 's youth academy in 1996 and went on to score 26 league goals in 76 appearances for the club before joining Corinthians in 2005 . Although Juventus are reluctant to lose the league 's top scorer - Tevez has netted 17 times this season - they have accepted the player wants to leave and have already started identifying potential replacements . Italian paper Tuttosport claims the 31-year-old has already decided to leave the club this summer Tevez has always said he intends to finish his career back at his first club Boca Juniors in Argentina Carlos Tevez 17 Mauro Icardi 16 Jeremy Menez 16 Tevez 's international manager Tata Martino claims any potential move would have no bearing on the player 's Argentina future . 'The most important thing for me is that Tevez continues to play as well as he is at the moment , ' Martino told Italian paper Tuttosport . 'It does n't matter whether that 's at Juventus or Boca . Only he 'll know what 's best for his future . ' I will say this , though : Juventus are a great team , but so are Boca Juniors . It would n't be a step down . '\n\nSummary:\n\nCarlos tevez has been told to terminate his contract with juventus to complete a return to his former club boca juniors in argentina . The former manchester city striker 's deal with the serie a champions does not expire until the end of next season but he has reportedly told the club he wishes to leave this summer . Tevez is on course to win a second serie a title with the old lady and still has a shot at european glory .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-02c955067d00f38b6978b805d5a8701a787f78ac",
+ "system_id": "M10",
+ "source": "Ten people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .",
+ "reference": "Three vehicles collided on the Brisbane Valley Highway , near Fernvale . A 40-year-old man with chest and shoulder injuries was airlifted to hospital . Also flown out was a six-year-old girl suffering from abdominal pain . Six others - including an infant and two young girls - had minor injuries . One person involved in the crash was found trapped inside their vehicle .",
+ "system_output": "Three vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nTen people , including four children , have been involved in a multi-vehicle crash in Brisbane 's west . Three vehicles collided on the Brisbane Valley Highway , 2km south of Fernvale , on Wednesday afternoon . A 40-year-old man with chest and shoulder injuries and a five-year-old boy with chest pain were the first patients flown out by rescue helicopter . They 've been taken to the Princess Alexandra Hospital in Brisbane . Shortly after , a 27-year-old woman with arm injuries and a six-year-old girl suffering from abdominal pain were also being flown out . The three car pileup happened at on the Brisbane Valley Highway , near Fernvale Six others - including an infant and two young girls - have been taken to Ipswich Hospital with minor injuries or for further assessment . A man has also suffered hand lacerations and two females escaped injuries . The affected stretch of the highway has been closed off , with diversions in place south of Fernvale . The Courier-Mail reports that earlier emergency crews found one person trapped inside their vehicle and Queensland Fire and Rescue crews used hydraulic cutting gear to free them . The Rescue 500 helicopter then landed at the scene and airlifted a number of patients to hospital . Police said the highway would be closed for a few hours with diversions in place south of Fernvale .\n\nSummary:\n\nThree vehicles collided on the brisbane valley highway , 2km south of fernvale . A 40-year-old man with chest and shoulder injuries and a five-year-old girl suffering from abdominal pain were also being flown out . A man has also suffered hand lacerations and two females escaped injuries .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M5",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Referee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nReferee mark clattenburg has been named to take charge of the manchester derby on sunday , despite having sent off three players from united and city this season . City captain vincent kompany was dismissed for two bookable offences during belgium 's narrow 1 - 0 defeat of israel in their euro 2016 qualifier on march 31 , meaning he is now suspended for the match against wales in june . Mark clattenburg will officiate the manchester derby between united and city at old trafford\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-808a439a5d83fc47baaea934c8779a7ebc62470d",
+ "system_id": "M23",
+ "source": "Referee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )",
+ "reference": "Manchester United host Manchester City in Premier League on Sunday . Mark Clattenburg has been named as the Manchester derby referee . Official sent off Vincent Kompany for Belgium and both his red cards shown in the league this season have been to United players .",
+ "system_output": "Mark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nReferee Mark Clattenburg has been named to take charge of the Manchester derby on Sunday , despite having sent off three players from United and City this season . City captain Vincent Kompany was dismissed for two bookable offences during Belgium 's narrow 1-0 defeat of Israel in their Euro 2016 qualifier on March 31 , meaning he is now suspended for the match against Wales in June . And , although Clattenburg has been accused of favouring Louis van Gaal 's side in the past , it 's worth noting that the 40-year-old has only sent off two players season in the Premier League this season and both have been from United ; Tyler Blackcett in the 5-3 defeat by Leicester and Luke Shaw in the 1-1 draw with West Ham . Mark Clattenburg will officiate the Manchester derby between United and City at Old Trafford The English referee sent off City and Belgium captain Vincent Kompany during the international break Leicester 5-3 Manchester United West Ham 1-1 Manchester United Manchester United 3-0 Tottenham Manchester City 3-1 West Ham Liverpool 2-1 Manchester City Chelsea 1-1 Manchester City Clattenburg has courted controversy during his career but is generally regarded as one of the Premier League 's leading referees alongside Michael Oliver . The champion 's shock 2-1 loss to Crystal Palace on Monday saw United move a point above their local rivals to add extra incentive for both sides ahead of the derby at Old Trafford , which could ultimately decide who finishes second behind expected winners Chelsea . While Manuel Pellegrini 's side have struggled since the turn of the year , turning from title challengers to fourth place chases , United are coasting on confidence having won their last five consecutive league games . Clattenburg will be joined on Sunday by assistants Simon Beck and Jake Collin , while Jonathan Moss will serve as the fourth official . Clattenburg has shown only two red cards this season , both to United players including Luke Shaw ( centre )\n\nSummary:\n\nMark clattenburg will take charge of the manchester derby on sunday . English referee has sent off three players from united and city this season . Vincent kompany was dismissed for two bookable offences for belgium . Tyler blackcett and luke shaw have also been sent off by clattenburg .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M11",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 3.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers , illinois , was born with cerebral palsy . She hopes to inspire others with her enthusiastic videos . She says her motivation for creating the video was to highlight stereotypes of people over certain disability . She has created a video in which she dances do n't judge : people tend to think that because i have cerebral palsy i can not do anything and that i am so different from everyone else , when in reality i am pretty much just the same as everyone else ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M15",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "Dayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nDayna dobias , 19 , from downers grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d3c9ce137ba24a69253508b3416630465c050ae4",
+ "system_id": "M9",
+ "source": "A teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area",
+ "reference": "Dayna Dobias , 19 , has created a video in which she dances despite having a disability that makes it difficult for her to walk . She loves TV , film and fashion , and says she \u2019 s not happy with how people with disabilities are represented . The teen has created several videos during the past year aimed at changing stereotypes .",
+ "system_output": "She hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 4.333333333333333,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA teenager from Illinois is tackling her disability head-on and attempting to positively influence thousands of others by dancing . Dayna Dobias , 19 , from Downers Grove was born with cerebral palsy , but she hopes to inspire others with her enthusiastic videos . ' I 've gotten bullied because of it and I get people all the time , staring and so it 's definitely made things difficult in my life , ' Dayna told Daily Mail Online . Scroll down for video Challenging : Dayna Dobias , 18 , is hoping change the way people with disabilities are represented by the television , film and the fashion industry Her hope is that the video not only entertains , but inspires others to think before judging . The teenager says her motivation for creating the video was to counteract stereotypes held by people over certain disabilities . 'People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' she added . ' I might do things a little bit differently than most people , but that does n't mean that I should be treated differently . ' This is one of several videos that Dayna has created . Last year the teenager produced another dancing clip with her strutting her stuff to Taylor Swift 's Shake It Off . Inspirational : The Illinois teen who suffers from cerebal palsy has created a video in which she dances Do n't judge : People tend to think that because I have cerebral palsy I can not do anything and that I am so different from everyone else , when in reality I am pretty much just the same as everyone else , ' Dayna says 'People like to define me based on disability . People just look at me and think , oh that 's just the girl with cerebral palsy , ' she said . ' I would like people to look past that and be like , oh she 's just one of us . ' Dayna says she wanted to turn any negativity that might be held over her disability into something positive and hopefully inspire others , or at the very least raise a smile . 'When I was first diagnosed with cerebral palsy , doctors warned my parents that I may never walk , but here I am dancing ! I drive , I go to college , I do things like every other 19-year-old , ' she says . The teen has now begun to do some motivational speaking at a couple of local schools . Confident : Dayna has now taken up motivational speaking and has visited several schools in the Chicago area\n\nSummary:\n\nShe hopes to inspire others with her enthusiastic videos . Teenager says her motivation for creating video was to counteract stereotypes . Dayna dobias , 18 , is hoping to change the way people with disabilities are represented by the television . Her hope is that the video not only entertains , but inspires others .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-4f3b35b540ba05f0f82d71a694ec23dde4dc3e01",
+ "system_id": "M1",
+ "source": "A gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson",
+ "reference": "Gang of at least three poured gasoline on a car at South LA gas station . Before the fire was started , the gang attempted to rob the driver of his car . Two people were inside white Dodge Charger when it went up in flames . No one was hurt and LA Fire Department is investigating crime as arson .",
+ "system_output": "The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.416666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA gang of at least three people poured gasoline on a car that stopped to fill up at California gas station early on Saturday morning and set the vehicle on fire . The driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 Station in South Los Angeles around 12:15am on Saturday . The group covered his white Dodge Charger in gasoline and lit it ablaze while there were two passengers inside . Scroll down for video Firefight : At least three people poured gasoline on a car and lit it on fire at a South Los Angeles gas station Explosive situation : Group covered the white Dodge Charger and lit it while there were two passengers inside Burnt out : The passengers and the driver were not hurt during the incident but the car was completely ruined The man 's grandmother said the fire was lit after the suspects attempted to carjack her grandson , ABC 7 reported . She said : 'He said he was pumping gas and some guys came up and asked for the car . 'They pulled out a gun and he took off running . 'They took the [ gas ] tank and started spraying . ' No one was injured during the fire , but the car 's entire front end was torched , according to NBC Los Angeles . The Los Angeles County Fire Department is investigating the incident as an arson and the suspects remain at large . Surveillance video of the incident is being used in the investigation . Heated argument : Before the fire , which occurred at 12:15am on Saturday , the suspects tried to carjack the man Hot case : The Los Angeles County Fire Department is investigating the incident at the LA 76 station as an arson\n\nSummary:\n\nThe driver of the car , who has not been identified , said he got into an argument with the suspects while he was pumping gas at a 76 station in south los angeles around 12:15 am on saturday . A gang of at least three people poured gasoline on a car that stopped to fill up at california gas station early on saturday morning and set the vehicle on fire . The group covered his white dodge charger in gasoline and lit it ablaze while there were two passengers inside .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2abe0f81f5b22c8a406165717e3bd7fce6fe47c6",
+ "system_id": "M17",
+ "source": "A southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa",
+ "reference": "Charles Manuel of Lamoni , Iowa admitted to a review board that he traded sexual favors for his services . Manuel also fessed up to performing exorcisms and to telling patients to stop taking medications prescribed to them by a medical doctor . The Iowa Board of Chiropractic required Manuel to pledge he would not apply for reinstatement of the license , but only for 10 years .",
+ "system_output": "Charles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA southern Iowa chiropractor accused of accepting sex as payment for his services and performing exorcisms on patients has surrendered his state license . The Iowa Board of Chiropractic released a report Wednesday detailing charges against Charles Manuel , of Lamoni . Manuel signed an agreement last month admitting his misdeeds and pledging not to apply for reinstatement for at least 10 years . Patient satisfaction : A chiropractor in Iowa has surrendered his license to practice and admitted to swapping services for sex and performing exorcisms on some patients . Nonetheless , he 's received outstanding evaluations for patient satisfaction on HealthGrades.com The agreement requires that Manuel prove any circumstances surrounding the revocation of his license no longer exist before he can resume practicing chiropractic in the state . Those circumstances included bartering sex for services with some patients . Manuel also recommended that patients stop taking medication he prescribed to them . A woman who answered a call to Manuel 's home from the Des Moines Register declined to comment on the case . A woman at his former practice said he had n't worked there for some time . A Lamoni address listed on the Yelp page for Manuel 's practice appears to be a home on a residential street . While Maneul has received just three patient survey responses on HealthGrades.com , those responses were quite positive . The disgraced chiropractor received a perfect five out of five stars in patient satisfaction . Strange practice : Charles Manuel , who admitted wrongdoing to the Iowa Board of Chiropractic , listed his practice 's addresses on this residential street in the small agricultural town of Lamoni in southern Iowa\n\nSummary:\n\nCharles manuel , of lamoni , iowa , admitted to the iowa board of chiropractic that he bartered services for sex with some patients . Manuel also admitted to performing exorcisms on patients and recommending they stop taking medication he prescribed to them . Manuel agreed not to apply for reinstatement of his license for at least 10 years .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M15",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 4.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . On tuesday , a u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian su-27 flanker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M12",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "Russian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 3.6666666666666665,
+ "relevance": 4.0,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nRussian fighter jet intercepted a u.s. reconnaissance plane in an `` unsafe and unprofessional manner `` earlier this week , the united states is complaining to moscow about the incident . The pentagon said the incident occurred in international airspace north of poland . Russian state news agency sputnik reported the u.s. plane was flying toward the russian border with its transponder switched off .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M8",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "The incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nThe incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed `` .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-5139ccfabee55ddb83e7937f5802c0a67aee8975",
+ "system_id": "M10",
+ "source": "( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .",
+ "reference": "The incident occurred on April 7 north of Poland in the Baltic Sea . U.S. says plane was in international airspace . Russia says it had transponder turned off and was flying toward Russia .",
+ "system_output": "A u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) After a Russian fighter jet intercepted a U.S. reconnaissance plane in an `` unsafe and unprofessional manner '' earlier this week , the United States is complaining to Moscow about the incident . On Tuesday , a U.S. RC-135U was flying over the Baltic Sea when it was intercepted by a Russian SU-27 Flanker . The Pentagon said the incident occurred in international airspace north of Poland . The U.S. crew believed the Russian pilot 's actions were `` unsafe and unprofessional due to the aggressive maneuvers it performed in close proximity to their aircraft and its high rate of speed , '' Pentagon spokesman Mark Wright said . Russian state news agency Sputnik reported the U.S. plane was flying toward the Russian border with its transponder switched off , according to a Defense Ministry spokesman . Maj. Gen. Igor Konashenkov said the Russian jet flew around the U.S. plane several times to identify it and get its tail number . An official with the U.S. European Command said the claim that the transponder was off was false . Wright said the Pentagon and State Department will `` file the appropriate petition through diplomatic channels '' with Russia . This is not the first time the U.S. has complained about an incident involving a RC-135U and a SU-27 . A year ago , a Russian jet flew within 100 feet of a RC-135U over the Sea of Okhotsk in the western Pacific , according to U.S. officials who called it `` one of the most dangerous close passes in decades . '' The Pentagon complained to the Russia military about that incident . Russian and U.S. aircraft often encounter each other , both in Northern Europe as well as the area between the Russian Far East and Alaska . CNN 's Steve Brusk and Jamie Crawford contributed to this report .\n\nSummary:\n\nA u.s. rc-135u was flying over the baltic sea when it was intercepted by a russian flanker . The pentagon said the incident occurred in international airspace north of poland . The u.s. crew believed the russian pilot 's actions were `` unsafe and unprofessional , `` pentagon says .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-051db270afc32571a08c15124330efe89d80fea8",
+ "system_id": "M9",
+ "source": "An FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office",
+ "reference": "Rifle was stolen overnight while agent 's car was parked at a Salt Lake City hotel across the street from the state 's FBI office . Gun was 'secured properly ' in a case and truck safe with padlocks and chains . Police believe thief tied a rope around the case and used another car to break the handle off by ripping the case through the car 's window . Agent 's stolen backpack and gear bags were recovered at a nearby hotel , but the gun has not been found .",
+ "system_output": "Fbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 4.0,
+ "overall": 3.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nAn FBI agent 's sniper rifle ( similar to the one pictured ) was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot An FBI agent 's sniper rifle was ripped out of his car 's window and stolen from a Salt Lake City hotel parking lot just days before President Obama visited Utah earlier this month . The gun was inside a hard rifle case and was 'secured properly ' to a truck safe with padlocks and chains while the car was parked at the Marriott Springhill Suites , according to police . Police believe the thief stole the rifle by breaking the rear right passenger-side window , tying a rope around the rifle case 's handle or a cable lock , and using the momentum from another vehicle to then break the case 's handle free from the lock . The window frame of the agent 's car was pulled from the door , which police believe could indicate the large case was ripped through the shattered window . 'The rifle had been secured properly , ' Salt Lake City Police Detective Dennis McGowan told Fox 13 Salt Lake City . 'There are all kinds of ways the weapon was secure . Yet someone was able to forcefully take the weapon . ' The FBI agent also reported that his gear bag , backpack , and some clothing and tools were also stolen during the March 27 theft , which the police report states happened overnight . The theft happened just a week before Obama ( pictured during his visit ) made a scheduled trip to Utah earlier this month Officers found two skid marks and pieces of rope lying on the pavement near the car , as well as the broken-off handle of the rifle case . The agent 's bags and some of his clothing were found at a Residence Inn across the street , but the sniper rifle has not been recovered . Marriott Springhill Suites , the agent 's hotel , is next to the city 's airport and just down the street from the state 's FBI office . The hotel does not have surveillance cameras , according to the police report . Salt Lake City police said the rifle 's serial number has been entered into the National Crime Information Computer . The Secret Service told Fox 13 Salt Lake City the agency was notified about the stolen rifle before President Obama 's visit , but could not comment if the theft prompted an increase of security . Det . McGowan said police do not know if the thief specifically targeted the agent 's car . The sniper rifle was stolen sometime overnight from the parking lot of Marriott Springhill Suites , where the agent was staying . The hotel is just across the street from the state 's FBI office\n\nSummary:\n\nFbi agent 's sniper was ripped out of his car 's window and stolen from a salt lake city hotel . The gun was inside a hard rifle case and was ` secured properly ' to a truck safe . Police believe the thief stole the rifle by breaking the rear right window . An fbi agent 's sniper rifle was ripped out of his car 's window and stolen from a salt lake city hotel just days before president obama visited utah earlier the window frame of the agent 's car was pulled from the door .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-b1f6faac3f406edca1e0cad8cce1319c83e62605",
+ "system_id": "M1",
+ "source": "The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .",
+ "reference": "Luigi Costa , 71 , is accused of killing his neighbour Terrence Freebody . Costa allegedly stomped on his head , cut his throat and stabbed him multiple times in Freebody 's dining room in Red Hill , Canberra on July 2012 . Experts says Costa suffered from dementia and alcohol abuse at the time . He also said there were signs of Costa 's decline in the lead-up to incident .",
+ "system_output": "Luigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThe man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Luigi Costa , 71 , is accused of stomping on his elderly neighbour Terrence Freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on Mugga Way in Red Hill , Canberra on July 2012 . Forensic psychiatrist Professor Paul Mullen examined Costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , The ABC reported . Luigi Costa , 71 , is accused of killing his elderly neighbour Terrence Freebody in the dining room of his home on Mugga Way , Red Hill , Canberra in July 2012 Professor Mullin told the court that the triple 0 calls which Costa made 'almost certainly ' during the incident proved that his mood fluctuated from 'moment to moment ' . He also said the trail of faeces from the crime scene to the toilet were police found Costa suggested his confused mental state . ' I believe that at the time of the killing Mr Costa 's brain was compromised by dementia and when you add to this alcohol that led to a confused state , ' he said . ' I do not think you would understand what you were doing in that state or understand the dreadful long-term consequences . ' Prosecutor Shane Drumgold suggested that Costa feigned his failing memory while being interviewed by Professor Mullin - who denied these claims . 'My view is that it was more likely than possible that Mr Costa was suffering a demented illness in the weeks , months and possibly years in the lead up to this event , ' he said . Costa 's state of mind is critical to case , as The jury can return a not guilty verdict if Costa 's unstable mindset at time of the attack can be proven . Sorry we are not currently accepting comments on this article .\n\nSummary:\n\nLuigi costa , 71 , is accused of stomping on his elderly neighbour terrence freebody 's head , cutting his throat and stabbing him multiple times in the dining room of his home on mugga way in red hill , canberra on july 2012 . The man who allegedly killed his neighbour was believed to be suffering from dementia and alcohol abuse at the time of the horrendous murder in 2012 , a jury has heard . Forensic psychiatrist professor paul mullen examined costa after the attack and believes there was evidence of the accused 's state of mind declining in the lead-up to the incident and also during the event , the abc reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c6667409fcdea2c89ba9f0d3a72c50e3967b4b1b",
+ "system_id": "M2",
+ "source": "Everton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley",
+ "reference": "Everton defeated Burnley 1-0 at Goodison Park on Saturday . Kevin Mirallas scored the only goal of the game in the 29th minute . Ross Barkley had earlier missed a 10th-minute penalty . Leighton Baines has scored 15 penalties from 16 attempts this season .",
+ "system_output": "Everton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.6666666666666665,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nEverton manager Roberto Martinez was forced to defend another penalty fiasco at the club after Ross Barkley missed from the spot in their 1-0 win against Burnley at Goodison Park . The untried Barkley inexplicably took the 10th minute kick \u2013 awarded for a foul by David Jones on Aaron Lennon \u2013 rather than Leighton Baines , who has scored 15 penalties from 16 attempts in the Premier League . Although there was no dispute between the team-mates this time , it brought back memories of Everton 's match against West Brom in January when Kevin Mirallas grabbed the ball from Baines to take a penalty - and missed . Ross Barkley steps up to take a 10th minute penalty despite the presence of Leighton Baines on the pitch Barkley 's effort is saved by Burnley goalkeeper Tom Heaton at Goodison Park Martinez insisted Barkley was within his rights to request penalty-taking duties on Saturday . 'If Romelu Lukaku had been on the pitch , he would have taken it . Otherwise , I am happy to have three or four players who can take penalties and let it depend on how they feel at that moment , ' argued the Everton manager . Baines ( left ) has scored 15 penalties from 16 attempts in the Premier League 'Ross showed incredible responsibility to take it . I love seeing players take control of the big moments and Leighton was happy to given him that responsibility . ' Barkley 's penalty was well-struck but was n't put in the corner and Burnley goalkeeper Tom Heaton dived to his right to save . Fortunately for the young England player , it did n't prove costly as Mirallas went on to score the only goal of the game after 29 minutes . Everton boss Roberto Martinez issues instructions to his players during a break in play against Burnley\n\nSummary:\n\nEverton manager roberto martinez was forced to defend another penalty fiasco at the club after ross barkley missed from the spot in their 1-0 win against burnley at goodison park . Barkley 's effort is saved by burnley goalkeeper tom heaton at goodison park . Martinez insisted barkley was within his rights to request penalty-taking duties on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-bcdb32ae644040a721da880fff9c1d502048cd8b",
+ "system_id": "M22",
+ "source": "( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .",
+ "reference": "Holland America says two passengers died in an apparent murder-suicide . `` The cabin was immediately secured and the authorities were notified , '' cruise line says . The FBI is investigating the deaths ; the ship is in San Juan , Puerto Rico .",
+ "system_output": "Holland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Two passengers found dead on a cruise ship in Puerto Rico appear to have died in a murder-suicide , the cruise line said . Holland America Line said two guests were found dead inside their stateroom on the ms Ryndam at 11:30 a.m. Thursday . `` The cabin was immediately secured , and the authorities were notified , including the FBI , '' Holland America said . `` We are cooperating fully with the investigation , and the authorities will make the official determination on what occurred . '' FBI spokesman Moises Qui\u00f1ones said authorities were on scene investigating . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . It 's currently in San Juan , Puerto Rico . Puerto Rico Port Authority spokesman Efra\u00edn Santiago told El Nuevo Dia newspaper that the cleaning staff on the ship had discovered the deceased passengers after knocking on the cabin 's door .\n\nSummary:\n\nHolland America Line says two guests were found dead inside their stateroom . The ship left Tampa , Florida , on March 29 on a 14-day Southern Caribbean cruise . Puerto Rico Port Authority spokesman Efra\u00edn Santiago says cleaning staff discovered the deceased passengers .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-46609841ffd23e9c22d1507edf7b176ecc4d834f",
+ "system_id": "M23",
+ "source": "Former Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .",
+ "reference": "Curacao have advanced to the second qualifying round for 2018 World Cup . Patrick Kluivert 's side won 4-3 on aggregate to set up match against Cuba . Curacao will face Cuba in next round on June 8 and 16 .",
+ "system_output": "Curacao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.166666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nFormer Holland forward and assistant coach Patrick Kluivert has made a winning start in World Cup qualifying as coach of Curacao . The Caribbean island team advanced to the second qualifying round in CONCACAF for the 2018 tournament late Tuesday . Curacao drew 2-2 in a second-leg match at Montserrat to win 4-3 on aggregate . Kluivert 's team won 2-1 on Saturday in Willemstad . Patrick Kluivert has helped Curacao progress to the next stage of 2018 World Cup qualification Former Holland assistant Kluivert ( far right ) worked with Louis van Gaal ( centre ) at the 2014 World Cup The first-round encounter was headed to extra time on Tuesday until former Holland Under 21 international Charlton Vicento scored in the 87th minute , five minutes after coming on as a substitute . Curacao face Cuba in the next round , on June 8 and 16 . Kluivert , whose mother is from Curacao , was a World Cup losing semi-finalist in 1998 as a player , and last year as assistant to coach Louis van Gaal .\n\nSummary:\n\nCuracao have qualified for the second qualifying round of the 2018 world cup . Curacao drew 2-2 in a second-leg match at montserrat to win 4-3 on aggregate . Patrick kluivert worked with louis van gaal at the 2014 world cup . Curacao face cuba in the next round on june 8 and 16 .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-2a48112ddee712f2364f128076cb2db555a043b1",
+ "system_id": "M17",
+ "source": "Esteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal",
+ "reference": "Esteban Cambiasso says saving Leicester will feel like winning a trophy . The Argentinian has become a key player for Nigel Pearson 's side . Leicester are currently seven points adrift at the bottom of the table .",
+ "system_output": "Esteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 3.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nEsteban Cambiasso has won all the major European competitions a player can during his illustrious career but revealed that keeping Leicester City in the Premier League would be up there with the best . The Foxes are currently seven points adrift at the bottom of the table , with only eight games remaining , knowing that time is running out to save themselves . Cambiasso refuses to give up and admits that keeping Leicester up will feel like winning a trophy . Esteban Cambiasso says that helping keep Leicester in the Premier League will feel like winning a trophy 'For me , it 's like another cup , ' he told BBC East Midlands Today . 'When you start another season you have an objective , and this is the objective for us . 'For me , winning a cup or winning the league with another team is the same now as having the possibility to save Leicester in the Premier League . ' The Argentinian midfielder poses with the trophy after his team won the 2010 FIFA Club World Cup Cambiasso had an illustrious career at Inter Milan , winning an impressive 15 trophies during his stint River Plate ( 2001-2002 ) Argentine Primera Division Real Madrid ( 2002-2004 ) La Liga Super Cup Supercopa de Espana Inter Milan ( 2004-2014 ) Champions League Serie A ( 5 ) Coppa Italia ( 4 ) Supercoppa ( 4 ) FIFA Club World Cup Having not won a game since January , Nigel Pearson 's men face West Ham United on Saturday and Cambiasso is still convinced they can avoid the drop . ' I understood when I signed for Leicester it 's not an easy job to stay in the Premier League , ' he said . 'It 's a difficult situation but I think we have our chances to win matches . There 's a quarter of the Premier League left to finish . ' I think some people think for Leicester all is finished . But I 'm sure , because I watch my team-mates every day , we can save Leicester and stay in the Premier League . ' The former Inter Milan star signed for the Foxes in the summer , leaving Italy after ten years and embarking on a new challenge in England . After agreeing to a one-year-deal , Cambiasso has quickly established himself as a key player but it remains to be seen if he 'll still be in the East Midlands at the start of next season . The former Real Madrid man was also successful during his short spell in Spain for Real Madrid Cambiasso played during Real 's 'Galatico ' era , with Luis Figo , Zinedine Zidane , Ronaldo and David Beckham 'Leicester always wanted me , ' he added . 'After these nine months or eight months , I 'm very happy because my family is OK , and I 'm good . ' I want a few more points , but all the rest is perfect . ' Cambiasso is happy in the East Midlands and could stay beyond his current one-year-deal\n\nSummary:\n\nEsteban cambiasso says keeping leicester in the premier league would be up there with the best achievements of his career . The foxes are currently seven points adrift at the bottom of the table , with only eight games remaining . Leicester face west ham united at upton park on saturday .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-fadabe346fe95d33eee71299e6596754768f5246",
+ "system_id": "M13",
+ "source": "A 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '",
+ "reference": "Gemma , 23 , has left her children to be raised by her mother . Handed them to her 52-year-old parent when they were four months old . Spends her time partying and has 'missed out ' on seeing them grow up . Is now at risk of being banned from seeing them thanks to feckless ways . Mother Debbie also says the unemployed 23-year-old has stolen from them .",
+ "system_output": "Gemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .",
+ "scores": {
+ "coherence": 2.3333333333333335,
+ "consistency": 2.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 3.3333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA 23-year-old mother-of-two is at risk of being banned from seeing her own children - because she refuses to stop drinking and partying . The woman , named only as Gemma , has two children under five by two different fathers and handed both infants over to her 52-year-old mother Debbie when they were four months old . Now Debbie is threatening to ban Gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , MKat , and because she suspects her of stealing . Scroll down for video Feckless : 23-year-old Gemma faces being prevented from seeing her children by her own mother ' I love them , ' says Debbie of the two children . 'She [ Gemma ] is n't fit enough to have them . She does n't care . ' Debbie , who stepped in to care for the children after social services threatened to take them away , says her daughter 's constant drinking and partying is getting out of hand . She also claims that Gemma , who is unemployed , stole an iPad from one of the children and sold it to fund her party lifestyle . Although Gemma says she did not steal the device , she admits to stealing money and other items from her mother in the past . 'From around the age of 13 , she went off the rails , ' says Debbie . 'She was drinking beer and having underage sex . Upset : Debbie reacts after hearing the results of Gemma 's lie detector test - which revealed she did steal Furious : Debbie , who has looked after Gemma 's children all their lives , says she will ban her from the house 'With the children , she was excited about being pregnant but gave up about four months after having them and then dumped the babies on me . ' As a result , Debbie , who appeared alongside her daughter on The Jeremy Kyle Show , has been left to bring both children up alone . Now at the end of her tether , Debbie says she is preparing to ban Gemma from her home and from seeing the children - because she ca n't trust her not pinch their possessions . Upset : According to her mother , Gemma prefers drinking and parties to work and motherhood ' I do n't know how you could do that to your own kid , ' adds Debbie , who also revealed that Gemma 's children call her mum and not grandma . 'She does n't care . She does n't care at all and I feel awful about it because at the end of the day , they 're my grandchildren and I love them . 'She 's missed out from day one . She does n't put them first . She knows she needs to grow up and stop drinking and taking drugs and partying . '\n\nSummary:\n\nGemma , 13 , has two children under five by two different fathers and handed both infants over to her 52-year-old mother debbie when they were four months old . Debbie is threatening to ban gemma from seeing the children at all , after discovering her daughter 's penchant for legal high , mkat , and because she refuses to stop drinking and partying . Debbie says she is preparing to ban gemma from her home and from seeing the children .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M12",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry had come in with a strong challenge from behind on united striker radamel falcao which sent the colombian hurtling to the ground .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M22",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Chelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nChelsea beat Manchester United 1-0 at Stamford Bridge on Saturday . Eden Hazard scored the opening goal of the game after John Terry 's challenge on Radamel Falcao . Thierry Henry , Jamie Redknapp and Graeme Souness agree Terry fouled the striker .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-f468efac7b3c54f8c42c2c81dff108c52ebe0d7d",
+ "system_id": "M8",
+ "source": "After Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao",
+ "reference": "Eden Hazard scored the opening goal for Chelsea against Manchester United . At the start of the preceding move , John Terry brought down Radamel Falcao . Sky Sports pundits agreed that play should have been stopped . Thierry Henry , Graeme Souness and Jamie Redknapp analysed at half-time .",
+ "system_output": "Eden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nAfter Chelsea forward Eden Hazard had scored give his side a 1-0 lead against Manchester United at Stamford Bridge on Saturday , the Sky Sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain John Terry had come in with a strong challenge from behind on United striker Radamel Falcao which sent the Colombian hurtling to the ground . Blues midfielder Cesc Fabregas then collected the loose ball and released it to Oscar before his back-heel sent it into the path of the oncoming Hazard to slot home . Chelsea defender John Terry comes in from behind with a strong challenge on Radamel Falcao Terry 's challenge on Falcao came during the build-up to Eden Hazard 's opening goal for Chelsea The Manchester United striker struggles to hold his grand and loses the ball to Terry near the half-way line Terry 's strong challenge leads to a loose ball that is picked up by Cesc Fabregas to start of the scoring move Without taking anything away from the sublime spectacle of the goal , pundits Thierry Henry , Jamie Redknapp and Graeme Souness agreed during their half-time analysis that play should have been stopped for Terry 's rough tackle on Falcao . Souness said , 'It 's a decent forearm smash . I mean he does stick his elbow in the back of Falcao 's head and that 's a foul . ' He added , 'If that elbow 's not there he 's not going down like that . ' Former Arsenal captain Henry , meanwhile , felt sympathy for Falcao as a fellow striker . Asked if he would have wanted a foul in that position as a player , the Frenchman responded , 'Yes , all day long . ' Finally , Redknapp admitted that while Falcao could have possibly done better to stay on his feet , Terry 's challenge from behind should n't have gone unpunished . ' I think he could have been stronger , but it 's still a foul . ' Sky pundits Graeme Souness ( left ) , Thierry Henry ( centre ) and Jamie Redknapp agree Terry fouled Falcao\n\nSummary:\n\nEden hazard scored give his side a 1-0 lead against manchester united . Sky sports punditry team were convinced that there had been a foul in the build-up . Chelsea captain john terry comes in with a strong challenge on radamel falcao .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-88c2481234e763c9bbc68d0ab1be1d2375c1349a",
+ "system_id": "M1",
+ "source": "( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .",
+ "reference": "Country band Lady Antebellum 's bus caught fire Thursday on a Texas freeway . A CNN iReporter captured the dramatic scene on video . Singer Hillary Scott shared a pic of the charred bus on Instagram .",
+ "system_output": "Michael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Lady Antebellum singer Hillary Scott 's tour bus caught fire on a Texas freeway Thursday morning , but everyone on board was safely evacuated . Michael Barnett captured dramatic video of the fire , on Interstate 30 just northeast of Dallas , and uploaded it to CNN iReport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and Barnett slowly approached in his vehicle . As he drew closer to the bus , Barnett decided to stop filming because he did n't know what to expect . `` It was shocking , '' he said . `` I did n't know what I was about to see . I did n't know if anyone was hurt . '' Barnett said he did n't realize at the time that the bus belonged to the country band . Hillary Scott , co-lead singer for the band , posted a photo of the charred bus on Instagram and noted that she , her husband , the tour manager and the driver were all evacuated safely . `` Thanking God for our safety and the safety of all of those who helped put this fire out and keep us safe , '' she wrote . The tour manager told CNN affiliate KTVT that the bus stopped after a rear tire blew out . It burst into flames after everyone had gotten off . Scott also posted an Instagram photo and message saying that the fire destroyed everything in the bus 's back lounge except her Bible . The band 's two other members , Charles Kelley and Dave Haywood , were not traveling on the bus , KTVT reported . Lady Antebellum is set to perform at the 50th Academy of Country Music Awards on Sunday in Arlington , Texas .\n\nSummary:\n\nMichael barnett captured dramatic video of the fire , on interstate 30 just northeast of dallas , and uploaded it to cnn ireport . Smoke and flames poured from the rear of the bus as traffic slowed to a crawl and barnett slowly approached in his vehicle . ( cnn ) lady antebellum singer hillary scott 's tour bus caught fire on a texas freeway thursday morning , but everyone on board was safely evacuated . Lady antebellum is set to perform at the 50th academy of country music awards on sunday in arlington , texas .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-6372eff07fd3d6f578f90d6115e01747edccc095",
+ "system_id": "M2",
+ "source": "Space lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .",
+ "reference": "Dr Adrian Quarterman has suggested mirrored satellites to collect sunlight . The beams can then be converted into laser light and sent down to earth . The physicist believes it could even make solar power feasible in Scotland . But he admits you might need to worry about who is in control of them .",
+ "system_output": "However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.333333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nSpace lasers are best known for blowing people - and planets - up in science fiction , and even the odd Bond movie . But one Scottish scientist is hoping they could , in fact , be the answer to heating our homes in the future . However , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : Dr Adrian Quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like Star Wars ( pictured ) Dr Adrian Quarterman , of Dundee University , is working on a laser which will convert sun beams into energy , The Times reported . The satellites will orbit the earth , covered in mirrors to help it harvest the sunlight and convert it into laser light . The energy it then creates , using semi-conductors , can be shot back down to earth . If it works , it could mean solar power could even become viable in Scotland , or be able to restore power instantly to disaster hit areas . The physicist told The Times : 'What this will do is be able to fire at solar panels to boost the energy they are receiving . ' Initial trials in Scotland were successful , but Dr Quarterman is now hoping to move the trials to Portugal . Similar ? : Tokyo rresearchers have proposed a laser system to attach to the ISS which would be used to shoot down pieces of debris in Earth orbit ( artist 's illustration shown ) Unlike the lasers in the movies , however , they will not be powerful enough to blind a pigeon , let alone blow up a planet . And it is quite different from ideas unveiled in a paper earlier this week , which could see the International Space Station fitted with lasers to clear up 'space junk ' . The system , suggested by researchers at the Riken research institute in Tokyo , would use a telescope already mounted on the station to find and target pieces of debris up to 62 miles ( 100km ) away . Powerful pulses from the laser would then push space junk into Earth \u2019 s atmosphere , where it would burn up . But this laser would also not be powerful enough to be turned into a weapon : the pieces it targets will be less than a centimetre in size . However , Dr Quarterman has acknowledged you 'might possibly have to worry a bit about who is in control ' of his satellites .\n\nSummary:\n\nHowever , he has n't ruled out the chance of someone could potentially use them for weapons . Space lasers : dr adrian quarterman has suggested space lasers could be used to power homes in the future - but says the beams will be nowhere near as powerful as imagined in films like star wars ( pictured ) . Dr adrian quarterman , of dundee university , is working on a laser which will convert sun beams into energy , the times reported .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M12",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "Rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 1.6666666666666667,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\nRochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men . She claimed donald sterling used the couple 's money to buy stiviano a ferrari . Stiviano countered that there was nothing wrong with donald sterling giving her gifts .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "cnn-test-404f859482d47c127868964a9a39d1a7645dd2e9",
+ "system_id": "M5",
+ "source": "( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .",
+ "reference": "V. Stiviano must pay back $ 2.6 million in gifts from Donald Sterling . Sterling 's wife claimed the ex-Clippers used the couple 's money for the gifts . The items included a Ferrari , two Bentleys and a Range Rover .",
+ "system_output": "( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\n( CNN ) Donald Sterling 's racist remarks cost him an NBA team last year . But now it 's his former female companion who has lost big . A Los Angeles judge has ordered V. Stiviano to pay back more than $ 2.6 million in gifts after Sterling 's wife sued her . In the lawsuit , Rochelle `` Shelly '' Sterling accused Stiviano of targeting extremely wealthy older men . She claimed Donald Sterling used the couple 's money to buy Stiviano a Ferrari , two Bentleys and a Range Rover , and that he helped her get a $ 1.8 million duplex . Who is V. Stiviano ? Stiviano countered that there was nothing wrong with Donald Sterling giving her gifts and that she never took advantage of the former Los Angeles Clippers owner , who made much of his fortune in real estate . Shelly Sterling was thrilled with the court decision Tuesday , her lawyer told CNN affiliate KABC . `` This is a victory for the Sterling family in recovering the $ 2,630,000 that Donald lavished on a conniving mistress , '' attorney Pierce O'Donnell said in a statement . `` It also sets a precedent that the injured spouse can recover damages from the recipient of these ill-begotten gifts . '' Stiviano 's gifts from Donald Sterling did n't just include uber-expensive items like luxury cars . According to the Los Angeles Times , the list also includes a $ 391 Easter bunny costume , a $ 299 two-speed blender and a $ 12 lace thong . Donald Sterling 's downfall came after an audio recording surfaced of the octogenarian arguing with Stiviano . In the tape , Sterling chastises Stiviano for posting pictures on social media of her posing with African-Americans , including basketball legend Magic Johnson . `` In your lousy f**ing Instagrams , you do n't have to have yourself with -- walking with black people , '' Sterling said in the audio first posted by TMZ . He also tells Stiviano not to bring Johnson to Clippers games and not to post photos with the Hall of Famer so Sterling 's friends can see . `` Admire him , bring him here , feed him , f**k him , but do n't put ( Magic ) on an Instagram for the world to have to see so they have to call me , '' Sterling said . NBA Commissioner Adam Silver banned Sterling from the league , fined him $ 2.5 million and pushed through a charge to terminate all of his ownership rights in the franchise . Fact check : Donald Sterling 's claims vs. reality CNN 's Dottie Evans contributed to this report .\n\nSummary:\n\n( cnn ) donald sterling 's racist remarks cost him an nba team last year . But now it 's his former female companion who has lost big . A los angeles judge has ordered v. stiviano to pay back more than $ 2.6 million in gifts after sterling 's wife sued her . In the lawsuit , rochelle `` shelly `` sterling accused stiviano of targeting extremely wealthy older men .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M15",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the barclays premier league title . Sky sports pundit thierry henry said : ' i think they need to buy four players - they need that spine . They need a goalkeeper , they still need a holding midfielder '\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M22",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Thierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 5.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nThierry Henry believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker in the summer . The Frenchman also questioned whether Olivier Giroud can win the Premier League with the Gunners . Arsenal are currently third in the table , level on points with Manchester City .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-d83b82a9cd504218e604616be3f3c3928e3428b8",
+ "system_id": "M10",
+ "source": "Thierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season",
+ "reference": "Thierry Henry hit out at Arsenal striker Olivier Giroud after Chelsea draw . The Sky Sports pundit does not believe Giroud can lead side to glory . Arsenal need four 'top quality ' stars to challenge for Premier League title . Henry 's former side drew 0-0 with Chelsea at the Emirates Stadium .",
+ "system_output": "Sky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 4.333333333333333,
+ "fluency": 4.333333333333333,
+ "relevance": 4.333333333333333,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThierry Henry has criticised Arsenal 's recent dealings in the transfer window and has urged his former side to purchase four top players in the summer . The Frenchman believes Arsenal need a goalkeeper , centre back , defensive midfielder and striker if they are to have any chance of winning the Barclays Premier League title . Henry , who was speaking after Arsenal 's goalless draw with Chelsea , also hit out at Olivier Giroud by claiming the Gunners will struggle to win the league if his fellow countryman is to remain as the club 's first choice striker . Sky Sports pundit Thierry Henry questioned whether Arsenal can win the title with Olivier Giroud in attack Henry ( centre ) said after Arsenal 's goalless draw with Chelsea that Giroud ( right ) may need replacing Hull ( away ) - May 4 Swansea ( home ) - May 11 Man United ( away ) - May 17 Sunderland ( home ) May 20 West Brom ( home ) - May 24 Speaking as a pundit on Sky Sports , Henry said : ' I think they need to buy four players - they need that spine . They need a goalkeeper , they still need a centre back , they still need a holding midfielder and , I 'm afraid , they need a top , top quality striker to win this league again . 'You had the best example with Chelsea here today . They were struggling last season to win the league . They went on to buy [ Thibaut ] Cortouis , [ Nemanja ] Matic , Cesc [ Fabregas ] and [ Diego ] Costa . 'For me , that speaks volumes and you saw it today - a team did n't and a team did , and they are about to win the league again . Arsenal are third in the Premier League , level on points with Manchester City but with a game in hand 'He [ Arsene Wenger ] has to do it . You need to have this competitive thing within the group - to win the league , that has to happen . ' And he added on Giroud 's inability to lead Arsenal to the title despite scoring 14 league goals so far this season : ' I think Giroud is doing extremely well . But can you win the league with him ? I would n't think so . 'He does a job , and he does it ever so well , but you ca n't win the league . 'Arsenal were at the top of the league last season , but everyone was saying they are not going win the league . ' Arsenal need to sign players of a similar calibre to Thibaut Courtois , Nemanja Matic , Cesc Fabregas and Diego Costa , according to Henry Giroud , pictured during Arsenal 's match against Chelsea , has scored 14 Premier League goals this season\n\nSummary:\n\nSky sports pundit thierry henry questioned whether arsenal can win the title with olivier giroud in attack . The frenchman believes the gunners need a goalkeeper , centre back , defensive midfielder and striker if they are to remain as the club 's first choice striker . Henry has urged his former side to purchase four top players in the summer .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M1",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "Jacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.416666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nJacob phillips , 23 , was found dead at the end of the disused northern promenade , a rocky area at the base of the cliff . Cardiff coroner 's court heard how the trio got out of the taxi to ` use an atm ' before making a run for it . The university graduate had been out drinking in cardiff before catching a taxi with friends padraig crowley and rory robinson .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-c50d33e9749e7bb484d9b69c4f5fca35a3a50cb5",
+ "system_id": "M0",
+ "source": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered",
+ "reference": "Jacob Phillips plunged down cliff after leaping over a fence in the dark . Got out of the taxi to 'use an ATM ' before running away , inquest heard .",
+ "system_output": "A young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate Jacob Phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr Phillips had caught the cab home after a night out with friends in December \u2013 but didn \u2019 t have enough cash to pay for it . An inquest heard he ran from the driver , who gave chase at the seaside town of Penarth , South Wales . Cabbie Dave Sidaway told the inquest : \u2018 I stopped because I couldn \u2019 t see where I was going . I had no idea how much danger he was in. \u2019 Jacob Phillips , 23 , was found dead at the end of the disused Northern Promenade , a rocky area at the base of the cliff The university graduate had been out drinking in Cardiff before catching a taxi with friends Padraig Crowley and Rory Robinson . Cardiff Coroner \u2019 s Court heard how the trio got out of the taxi to \u2018 use an ATM \u2019 before making a run for it . \u2018 The plan was to go to Penarth but we did not have enough money for the taxi \u2013 I ran off in one direction , Jacob went in another direction , \u2019 Mr Robinson told the inquest . \u2018 Running away from the taxi was a stupid thing to do. \u2019 Mr Phillips , of Reading , Berks , was discovered the next day on the rocky beach by a dog walker . Paramedics arrived at the scene after his body was discovered in the morning by a dog walker A long way down : Mr Phillips fell down this cliff after a night out . Tthe taxi driver said he would not have left him if he knew where they were Recording a verdict of accidental death , coroner Andrew Barkley said : \u2018 It is likely that he became disorientated and there is no evidence that he knew where he was at the time . \u2018 There is no way he could have known the danger that lay in front of him . The following day with the benefit of light it was clear the danger he faced by the sheer drop off the cliff just yards beyond the fence. \u2019 Mr Phillips worked for Pretty Green , the clothing label run by Oasis star Liam Gallagher . He had grown up in Barry , South Wales , before studying accountancy at the University of Birmingham . The inquest heard that he had returned to Wales for Christmas and had been staying with his mother in the upmarket Cowbridge area . Mr Phillips had planned to stay at a friend \u2019 s house but he and his friends didn \u2019 t have enough money to complete the journey , so decided to trick the driver by \u2018 dodging \u2019 the fare . Mr Phillips ran off into the \u2018 pitch black \u2019 and fell down a cliff at the end of the disused Northern Promenade \u2013 also called The Dardanelles . Police search the scene at the bottom of the cliff where his body was discovered\n\nSummary:\n\nA young accountant fell down a 70ft cliff to his death when he ran from a taxi to avoid paying the fare . Graduate jacob phillips , 23 , plunged down the cliff when he leapt over a fence in the dark . Mr phillips had caught the cab home after a night out with friends in december -- but did n't have enough cash to pay for it .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-2377095dda83df778e34304b7a09813808105d3b",
+ "system_id": "M9",
+ "source": "Thousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .",
+ "reference": "HM Passport Office struggled to cope with 3.6million Britons applications . Ministers agreed to give urgent cases a free upgrade to fast-track service . Only 2,191 compensation applications were approved totalling \u00a3203,066 . Meanwhile managers at agency were handed total of \u00a31.8million in bonuses .",
+ "system_output": "Managers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .",
+ "scores": {
+ "coherence": 1.3333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.0,
+ "overall": 3.583333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nCoherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby \"the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic.\"\n\nEvaluation Steps:\n\nDetermine if the summary is well-structured and well-organized.\nRate the summary on a scale of 1-5 for Coherence, where:\n\t* 1 = Incoherent or disorganized summary\n\t* 2 = Somewhat disorganized or lacking clear structure\n\t* 3 = Neutral or average coherence\n\t* 4 = Well-organized and coherent summary\n\t* 5 = Exceptionally well-structured and coherent summary\nAssess the collective quality of all sentences in the summary.\nDetermine if the summary is well-structured and well-organized.\nDecide if the summary builds from sentence to sentence to form a coherent body of information about a topic.\nRead the provided summary carefully.\n\n\nExample:\n\n\nSource Text:\n\nThousands of holidaymakers caught up in last summer \u2019 s passport delay fiasco have not received any compensation . At the same time , managers at the passport agency , which made a \u00a342 million profit during the chaos , received up to \u00a33,500 in bonuses . Ministers refused to give a blanket refund to the desperate families who had to pay extra to get their travel documents rushed through , and even to some who missed out on trips . Scroll down for video The boxes of passport applications piled up at in office in Liverpool at the peak of the backlog last summer Keith Vaz , who as chairman of the Home Affairs Committee led the calls for compensation , said last night : \u2018 I am astonished so few people have been given compensation for what was a fiasco presided over by the management of the passport office , especially as they made a profit last year that ran into millions of pounds. \u2019 The problems began a year ago as HM Passport Office struggled to cope with 3.6 million Britons applying for documents . By mid-June there were more than half a million passports stuck in the system . Ministers agreed to give urgent cases a free upgrade to the fast-track service , but thousands had already paid the extra \u00a330 per passport for this . Many still missed trips because their passports did not arrive in time . Details obtained by The Mail on Sunday show scarcely any of the thousands of holidaymakers caught up in the chaos ever got anything back from the Passport Office . In total just 2,191 compensation applications were approved between April last year and January this year , scarcely changed from 2,077 for the previous year . The total paid out was \u00a3203,066 , giving an average sum of \u00a392.68 . The biggest single payout was \u00a35,463 ; the lowest \u00a31 . The Passport Office said it could not say how many claims were rejected , partly because of \u2018 the system failure of our customer complaint database \u2019 . It can be revealed however that Ministers rejected a demand by the Home Affairs Select Committee for all those left out of pocket to be compensated , saying : \u2018 It would create a precedent. \u2019 Home Affairs Committee chairman Keith Vaz led the calls for compensation and was 'astonished ' to learn just 2,191 compensation applications were approved This newspaper contacted more than a dozen holidaymakers who had either missed trips or had to pay extra to get passports in time . Only one had got money back . Among those refused were Mathew Bean and Hayley Kirkham . They lost close to \u00a31,500 on a trip to Morocco , where he had planned to propose , because their passports did not arrive in time \u2013 even though they had applied ten weeks before they were due to travel and paid for upgrades . HM Passport Office made a surplus of \u00a342.3 million between April and October last year . In 2013-14 , the most recent figures available , managers were handed a total of \u00a31.8 million in bonuses , with the average reward \u00a3499 and the highest \u00a33,500 .\n\nSummary:\n\nManagers at the passport agency received # 42 million profit during the chaos . Ministers refused to give a blanket refund to the desperate families who missed out on trips . The boxes of passport applications piled up at the peak of the backlog last summer . Problems began a year ago as hm passport office struggled to cope .\n\n\nEvaluation Form (scores ONLY):\n\n- Coherence: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ }
+]
\ No newline at end of file
diff --git a/results/llama/scrambled_con.json b/results/llama/scrambled_con.json
new file mode 100644
index 0000000000000000000000000000000000000000..1efc856491bf8b56c4f9e339291435d6cee94132
--- /dev/null
+++ b/results/llama/scrambled_con.json
@@ -0,0 +1,1602 @@
+[
+ {
+ "doc_id": "dm-test-8764fb95bfad8ee849274873a92fb8d6b400eee2",
+ "system_id": "M8",
+ "source": "Paul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .",
+ "reference": "Andros Townsend an 83rd minute sub in Tottenham 's draw with Burnley . He was unable to find a winner as the game ended without a goal . Townsend had clashed with Paul Merson last week over England call-up .",
+ "system_output": "Paul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 3.0,
+ "fluency": 4.666666666666667,
+ "relevance": 2.3333333333333335,
+ "overall": 2.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nPaul Merson has restarted his row with Andros Townsend after the Tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with Burnley on Sunday . 'Just been watching the game , did you miss the coach ? # RubberDub # 7minutes , ' Merson put on Twitter . Merson initially angered Townsend for writing in his Sky Sports column that 'if Andros Townsend can get in ( the England team ) then it opens it up to anybody . ' Paul Merson had another dig at Andros Townsend after his appearance for Tottenham against Burnley Townsend was brought on in the 83rd minute for Tottenham as they drew 0-0 against Burnley Andros Townsend scores England 's equaliser in their 1-1 friendly draw with Italy in Turin on Tuesday night The former Arsenal man was proven wrong when Townsend hit a stunning equaliser for England against Italy and he duly admitted his mistake . 'It 's not as though I was watching hoping he would n't score for England , I 'm genuinely pleased for him and fair play to him \u2013 it was a great goal , ' Merson said . 'It 's just a matter of opinion , and my opinion was that he got pulled off after half an hour at Manchester United in front of Roy Hodgson , so he should n't have been in the squad . 'When I 'm wrong , I hold my hands up . I do n't have a problem with doing that - I 'll always be the first to admit when I 'm wrong . ' Townsend hit back at Merson on Twitter after scoring for England against Italy Sky Sports pundit Merson ( centre ) criticised Townsend 's call-up to the England squad last week Townsend hit back at Merson after netting for England in Turin on Wednesday , saying 'Not bad for a player that should be 'nowhere near the squad ' ay @ PaulMerse ? ' Any bad feeling between the pair seemed to have passed but Merson was unable to resist having another dig at Townsend after Tottenham drew at Turf Moor .\n\nSummary: \n\nPaul merson has restarted his row with andros townsend after the tottenham midfielder was brought on with only seven minutes remaining in his team 's 0-0 draw with burnley . Townsend was brought on in the 83rd minute for tottenham as they drew 0-0 against burnley . Paul merson had another dig at andros townsend after scoring for england against italy .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M14",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard slipped on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The entire incident was caught on camera .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M5",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "The queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.083333333333333
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nThe queen 's guard was left red - faced after he slipped on a manhole cover during the popular changing of the guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The queen 's guard ( pictured ) slipped on a manhole cover during the popular changing of the guard at buckingham palace last week .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0f0789390d67698283cc87b2e046b5c5cd77edb7",
+ "system_id": "M8",
+ "source": "This is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard",
+ "reference": "Buckingham Palace guard slipped and fell in front of hundreds of tourists . Thought to have stumbled on manhole cover during Changing of the Guard . Embarrassed young soldier ended up on the floor still clutching his rifle . Unfortunately for him the entire incident was caught on tourist 's camera .",
+ "system_output": "Queen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.25
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nThis is the embarrassing moment a Buckingham Palace guard slipped and fell on a manhole cover in front of hundreds of shocked tourists as he took up position in his sentry box . The Queen 's Guard was left red-faced after he slipped on a manhole cover during the popular Changing of the Guard - and unfortunately for him the entire incident was caught on camera . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The Queen 's Guard ( pictured ) slipped on a manhole cover during the popular Changing of the Guard at Buckingham Palace last week . Unfortunately for him , the entire incident was caught on a tourist 's camera The embarrassed soldier quickly scrambled to his feet as his colleagues marched past as if nothing had happened . But the young guard started to blush as he peered at the crowd from under his bearskin and realised how many people had seen his slapstick moment . Holidaymaker David Meadwell recorded the unscheduled manouevre outside Buckingham Palace on Thursday afternoon . Mr Meadwell , 50 , from Newcastle-upon-Tyne , said : ' I was with my family for a trip to London and I thought I 'd take some pictures of the changing of the guards . Tourist David Meadwell shot this footage of the Changing of the Guard last week when the incident unfolded The moment it all started to go wrong : The guard leans heavily to the side as he appears to slip The unidentified young guard 's legs appear to get a bit tangled as he tries to turn to march away The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip up 'The first group changed successfully , without any problems , and so I decided to video the next group doing it . ' I did n't expect anything like this to happen - he went flying . There were quite a few people around and there were a lot of gasps as he went down . ' I think he just slipped on a manhole cover , he looked so embarrassed . ' The unnamed solider is thought to have slipped because of the metal protectors nailed to the soles of his boots . Tourists gather in their thousands to watch the changing of the guard outside Buckingham Palace at 11.30am every day in the summer and every other day in the winter . The Guard comprises two detachments , one each for Buckingham Palace and St James 's Palace , under the command of the Captain of The Queen 's Guard . Contrary to popular belief they are not purely ceremonial and are fully operational soldiers . The Ministry of Defence said they would not comment on ' a young man falling over while doing his job ' . The embarrassed guard hastily scrambled to his feet following the unfortunate tumble at the palace The incident took place in front of hundreds of tourists who were watching the Changing of the Guard\n\nSummary: \n\nQueen 's guard slipped and fell on a manhole cover during the popular changing of the guard at buckingham palace last week . He lost his footing and slid sideways , knocking his bearskin on the side of the box and dropping his rifle . The guard , wearing full regalia , falls heavily to the floor still clutching his rifle following the slip .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-fbbafa743a8c2ecd2cedf65c6c61956b2db8ec5c",
+ "system_id": "M13",
+ "source": "( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''",
+ "reference": "The A & E networks are remaking the blockbuster `` Roots '' miniseries , to air in 2016 . The epic 1977 miniseries about an African-American slave had 100 million viewers .",
+ "system_output": "`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\n( CNN ) One of the biggest TV events of all time is being reimagined for new audiences . `` Roots , '' the epic miniseries about an African-American slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . Now A & E networks are remaking the miniseries , to air in 2016 . A & E , Lifetime and History ( formerly the History Channel ) announced Thursday that the three networks would simulcast a remake of the saga of Kunta Kinte , an African who was captured , shipped to America and sold into slavery to work on a Virginia plantation . LeVar Burton , who portrayed Kinte in the original , will co-executive produce the new miniseries . A press release describes the new version as `` original '' and `` contemporary '' and will draw more from Alex Haley 's classic novel , `` Roots : The Saga of an American Family . '' Producers will consult scholars in African and African-American history for added authenticity . `` We are proud to bring this saga to fans of the original , as well as to a new generation that will experience this powerful and poignant tale for the first time , '' said Dirk Hoogstra , History 's executive vice president and general manager . `` Audiences will once again feel the impact of Kunta Kinte 's indomitable spirit . '' Executive producer Mark Wolper , son of the original 's producer David L. Wolper , added , `` Kunta Kinte began telling his story over 200 years ago and that story went through his family lineage , to Alex Haley , to my father , and now the mantle rests with me . Like Kunta Kinte fought to tell his story over and over again , so must we . '' The remade `` Roots '' will encounter a new generation of viewers who have witnessed Barack Obama make history as the nation 's first African-American president and `` 12 Years a Slave '' win the Oscar for Best Picture , but also widespread racial unrest over police treatment of black suspects in many U.S. cities . `` My career began with 'Roots ' and I am proud to be a part of this new adaptation , '' said Burton . `` There is a huge audience of contemporary young Americans who do not know the story of 'Roots ' or its importance . ''\n\nSummary: \n\n`` roots , `` the epic miniseries about an african-american slave and his descendants , had a staggering audience of over 100 million viewers back in 1977 . A & e networks are remaking the miniseries , to air in 2016 . Levar burton , who portrayed kinte in the original , will co-executive produce the new miniseries .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M20",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Alleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .",
+ "scores": {
+ "coherence": 3.0,
+ "consistency": 1.3333333333333333,
+ "fluency": 3.0,
+ "relevance": 4.333333333333333,
+ "overall": 2.9166666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nAlleged abuse by Clarence House staff took place in 1978 when he was 27 . He was accosted after he walked into a room to look for a pen on his first shift He said he was told by police that he had 'failed to establish the identity of the accused ' and the case was closed . He said : ' I was devastated .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-f37fd6e9b6cc18a7132568e307ef3b130931e809",
+ "system_id": "M8",
+ "source": "Police are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .",
+ "reference": "Christopher Lawler claims he was pinned to a chair and groped by staff . He left job at Clarence House the same day as alleged incident in 1978 . Mr Lawler , now 68 , claims he reported his account but was twice ignored . The palace is now 'cooperating ' with police to investigate historic claims .",
+ "system_output": "Christopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nPolice are investigating claims by a former royal footman that palace aides tried to force him into an orgy , it was revealed yesterday . Christopher Lawler said he was pinned to a chair and groped by a male member of staff on his first day working at Clarence House . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a VIP paedophile ring in the 1970s and 80s . Police are investigating claims made by a former royal footman that Clarence House aides tried to force him into an orgy in the 1970s But his complaints were ignored for months before the police were finally notified , he claims . Mr Lawler , now 64 , said the attempted abuse at the Queen Mother \u2019 s London residence happened in January 1978 when he was 27 . He said he was accosted after he walked into a bedroom to look for a pen on his first shift . Two men offered him a drink before two other men joined them \u2013 one a senior member of staff , he said . He was asked if he was gay before a younger man took his trousers off and began performing a sex act on himself . \u2018 That prompted another guy to put his hand on my leg and then he grabbed me , \u2019 he told the Sunday People . \u2018 I was staggered . The younger man then came up behind me and gripped me , holding me in the chair . Mr Lawler worked at Clarence House when the Queen Mother used it as her London residence \u2018 They were trying to undo my trousers but I managed to jump up and burst out of the room. \u2019 Afterwards , Mr Lawler said he was followed by two men and threatened to keep quiet . A complaint he made that day allegedly drew an angry response from Clarence House \u2013 so he packed his bags immediately . Mr Lawler , a former Port of Liverpool police officer , said he was \u2018 haunted \u2019 by the incident for years . After hearing about the probe into historical cases of child abuse last year he wrote to the Palace , but was twice rebuffed . Months later the complaint was finally passed to Scotland Yard . A retired royal aide , who is now in his 80s , was reportedly interviewed but denied he was involved because he was working for the Queen in Balmoral at the time . Mr Lawler has now been told by the Palace that the Royal Household would work \u2018 exhaustively and co-operatively \u2019 with any police probe . A police spokesman said it would be inappropriate to comment \u2018 as investigations continue \u2019 .\n\nSummary: \n\nChristopher lawler said he was pinned to a chair and groped by a male member of staff on his first day working at clarence house . The ordeal left him in tears and he left the job the same day . He finally decided to contact palace officials again last year after inquiries began into alleged cover-ups of child sex abuse by a vip paedophile ring in the 1970s .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-2c37d44d03ce2e91310339d884d33ee5aabf9abc",
+ "system_id": "M10",
+ "source": "A two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )",
+ "reference": "The child was held by his mother when he slipped and fell between 10 and 12ft into the pit on Saturday around 3pm at the Cleveland Metroparks Zoo . He was rescued by his parents before emergency responders arrived on the scene ; he suffered from minor bruises and bumps . The cheetahs seemed to ignore the boy and his parents while in the pit . Zoo plans to press child endangerment charges .",
+ "system_output": "The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .",
+ "scores": {
+ "coherence": 4.666666666666667,
+ "consistency": 4.666666666666667,
+ "fluency": 5.0,
+ "relevance": 5.0,
+ "overall": 4.833333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nA two-year-old boy is recovering after falling into a cheetah exhibit at the Cleveland Metroparks Zoo after his parents dangled him over the edge , officials said . The toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on Saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . Scroll down for video A mother was holding the two-year-old boy and another child when the toddler slipped and fell into the cheetah exhibit at the Cleveland Metroparks Zoo ( file photo of cheetahs at the Cleveland zoo ) The boy was rescued by his parents from the pit ( pictured ) before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital He is listed in stable condition after being taken to MetroHealth Medical Center and suffered from minor bruises and bumps . The boy 's leg was injured in the fall , but he was not attacked by the animals , Dr. Christopher Kuhar , the zoo 's executive director told Fox 8 . Michael Lurie and his family were at the Cheetah exhibit when they heard the child scream . 'You saw how far the drop was and you just could n't believe the kid did n't hurt himself from falling down on the ground , ' Lurie told WKYC . ' I was just shocked , ' he said . ' I did n't understand how the parents let the kid go over the thing . ' The cheetahs did not approach the boy or his parents while in the pit , according to zoo officials . Zoo visitor Terra Lurie believes the boy was not approached by the fast feline because they were frightened . ' I think they were just curious as to what was going on and why somebody was in the pen with them , ' she said . 'It 's not every day that somebody is just in the pen with them . ' 'And everyone else is screaming and they probably got scared . ' Kuhar said the zoo had received ' a number of eyewitness accounts ' that indicate the 'strong likelihood that the child was dangled over the railing , ' he told NewsNet5 . Cleveland Metroparks Zoo has plans to press child endangerment charges against the family on Monday . The exhibit was closed following the child 's fall . Zoo visitor Michael Lurie was at the cheetah exhibit when he heard the child scream . He said he was 'shocked ' and 'didn't understand how the parents let the kid ' go over the railing and into the pit Cleveland Metroparks Zoo plans to press child endangering charges against the child 's mother ( above file photo of visitors at the Cleveland zoo )\n\nSummary: \n\nThe toddler 's mother was holding him and another child when he slipped and fell between 10 to 12ft and into the pit on saturday around 3pm . The child was rescued by his parents before firefighters and paramedics arrived on the scene . He suffered from minor bumps and bruises and was listed in stable condition at the hospital .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-e428e25bbf8e06643154ce31b7c6fab64c81e857",
+ "system_id": "M1",
+ "source": "The owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .",
+ "reference": "Speed camera discovered pointing at house in Handsworth , Birmingham . Fixed cameras switched off across the West Midlands in Spring of 2013 . Site is not going to be part of a new trial using digital technology . Obsolete camera may now be taken down after engineers examine device .",
+ "system_output": "The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .",
+ "scores": {
+ "coherence": 3.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 4.166666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nThe owners of this house better not leave too quickly , after a speed camera was angled directly on their front door . The bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in Handsworth , Birmingham . However , it has not been working for two years after every single fixed device was switched off in the West Midlands . Big Brother is watching : A speed camera has been turned round and is pointing at this house in Birmingham , West Midlands The speed camera has not been working for more than two years Around 300 speed and traffic camera , using old technology , were turned off across the region in March 2013 In there place , speed enforcement operations have been carried out by a small number of mobile camera units , fixed cameras on motorways and traffic officers on patrol . Mystery surrounds who had re-pointed the camera , but a spokesman for Birmingham City Council said they were aware of it . One of their engineers will now be visiting the site and the camera could be removed completely . 'Fixed location safety cameras have been decommissioned across the West Midlands since 2013 as the technology inside them had become obsolete , ' the spokesman said . 'Plans for a pilot at a limited number of sites , using digital technology , is currently in development . 'Now the issue with this camera in Wellington Road has been brought to our attention , we will take any appropriate action at the site . ' The spokesman confirmed that there were no plans to include the camera in Wellington Road in the new pilot . The owners of the house were not available for comment .\n\nSummary: \n\nThe bright yellow gatso had previously enforced the 30mph speed limit for motorists along the residential road in handsworth , birmingham . However , it has not been working for two years after every single fixed device was switched off in the west midlands . Around 300 speed and traffic camera , using old technology , were turned off across the region in march 2013 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M13",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .",
+ "scores": {
+ "coherence": 3.3333333333333335,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.333333333333334
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt reacts as he wins 4x100m gold at the london olympic games in 2012 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-0783c68a744df6afa009d16e72709f2b4d90dea1",
+ "system_id": "M14",
+ "source": "Usain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours",
+ "reference": "Usain Bolt will compete at the IAAF/BTC World Relays on May 2-3 . Six-time Olympic gold medalist says he 's 'fit , healthy and ready to run . ' Bolt was part of the Jamaica team that won gold at London 2012 .",
+ "system_output": "Usain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.666666666666667,
+ "overall": 4.666666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nUsain Bolt will compete at the IAAF/BTC World Relays in the Bahamas next month , the Jamaica Athletics Administrative Association has announced . The six-time Olympic gold medallist will compete at the relay championship on May 2 and 3 as part of the Jamaican team . ' I 'm happy to be part of the Jamaican team for the IAAF / BTC World Relays in the Bahamas . I am fit , healthy and ready to run , ' said Bolt . Usain Bolt has confirmed he will be part of Jamaica 's team at the World Relays in the Bahamas Bolt reacts as he wins 4x100m gold at the London Olympic Games in 2012 ' I hear the meet was a lot of fun last year and there was a great atmosphere . Jamaica has a long and successful tradition in relays and when we put on the national colours we always do our best to make the country proud , ' he added . JAAA General Secretary Garth Gayle commented , 'We were extremely pleased that Usain was available for selection and that the world 's fastest man will be running for Jamaica . We can expect some sprint magic on the track in the Bahamas on 2nd and 3rd May . ' The full Jamaican team list for the competition will be announced shortly . Bolt insists he always does 'his best to make his country proud ' while wearing Jamaica colours\n\nSummary: \n\nUsain bolt will compete at the iaaf/btc world relays in the bahamas . The six-time olympic gold medallist will compete at the relay championship on may 2 and 3 as part of the jamaican team . Bolt insists he always does ` his best to make his country proud ' .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M1",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "However real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 2.6666666666666665,
+ "overall": 3.5833333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nHowever real madrid remain top of their euro league division over their bitter rivals , just by points difference . Team-mates neymar and dani alves proved their dedication to barcelona by supporting the club 's basketball side . Barca prevailed with a narrow 85-80 victory in the euro league contest .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M8",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.916666666666667
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and alves headed to watch el clasico on thursday night . Dani alves proved their dedication to barcelona by supporting basketball side . Real madrid remain top of their euro league division over their bitter rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "3"
+ },
+ {
+ "doc_id": "dm-test-eeef09d26cf30c2124c0399b08eedc6321fe5d20",
+ "system_id": "M9",
+ "source": "Team-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday",
+ "reference": "Neymar helped Brazil beat Chile 1-0 at the Emirates stadium last weekend . Barcelona won the El Clasico to go four points clear at the top of La Liga . Luis Enrique 's side take on Celta Vigo in La Liga on Sunday .",
+ "system_output": "Neymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 5.0,
+ "fluency": 4.666666666666667,
+ "relevance": 3.3333333333333335,
+ "overall": 3.7500000000000004
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nTeam-mates Neymar and Dani Alves proved their dedication to Barcelona by supporting the club \u2019 s basketball side . Neymar and Alves headed to watch El Clasico on Thursday night alongside the Brazilian 's sister Rafaella . Barca prevailed with a narrow 85-80 victory in the Euro League contest . Brazil star Neymar ( centre ) takes a selfie with friends and Barcelona team-mate Dani Alves ( right ) However Real Madrid remain top of their Euro League division over their bitter rivals , just by points difference . Neymar helped Brazil beat Chile 1-0 at the Emirates Stadium on Sunday in a feisty contest and had to withstand several brutal challenges from the South American rivals . Before the international break Luis Enrique 's Barcelona had won their El Clasico contest to move four points clear at the top of La Liga . Neymar and his Barcelona team-mates return to La Liga on Sunday , taking on Celta Vigo as they continue to compete for a treble of trophies . Neymar 's sister Rafaella ( left ) headed to watch El Clasico of basketball with the Barcelona forward Neymar 's sister Rafaella ( right ) attends a Euro League basketball match between Barcelona and Real Madrid Neymar is distracted at the basketball as his sister Rafaella ( centre ) watches on Neymar , Brazil 's captain , led his national side to a 1-0 win over Chile at the Emirates last weekend Barcelona team-mate Dani Alves ( front left ) joined Neymar and friends to watch some live basketball Gary Medel walks off having appeared to stamp on Neymar at the Emirates on Sunday\n\nSummary: \n\nNeymar and dani alves proved their dedication to barcelona . Neymar and alves headed to watch el clasico on thursday night . Neymar helped brazil beat chile 1-0 at the emirates stadium on sunday . Real madrid remain top of their euro league division over their rivals .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M5",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Rome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .",
+ "scores": {
+ "coherence": 4.333333333333333,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.333333333333333,
+ "overall": 4.666666666666666
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nRome ( cnn ) italy is coping with a rising wave of desperate migrants from africa and middle east hoping to make it to europe . From friday to monday , a total of 8,480 migrants were rescued , according to the italian coast guard , which said it received on monday -- alone -- sos calls from 20 boats in distress .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "cnn-test-b1c3fc03a2b74cf4c79844c1fe2fdce70a8a436e",
+ "system_id": "M10",
+ "source": "Rome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .",
+ "reference": "The Italian coast guard says 8,480 migrants were rescued from Friday to Monday . Save the Children said Tuesday 400 migrants could be missing from a boat . The Italian coast guard can not confirm that report .",
+ "system_output": "Italian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .",
+ "scores": {
+ "coherence": 2.0,
+ "consistency": 2.3333333333333335,
+ "fluency": 5.0,
+ "relevance": 2.0,
+ "overall": 2.8333333333333335
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nRome ( CNN ) Italy is coping with a rising wave of desperate migrants from Africa and Middle East hoping to make it to Europe . From Friday to Monday , a total of 8,480 migrants were rescued , according to the Italian coast guard , which said it received on Monday -- alone -- SOS calls from 20 boats in distress . On Tuesday , a spokesman with Save the Children told CNN the group fears 400 migrants could be missing , citing testimony from survivors who said their ship carrying 550 people capsized in the Mediterranean Sea about 80 miles off the Libyan coast . The Italian coast guard , however , told CNN that while it is taking the report seriously , it can not confirm such an incident and has not yet found evidence at sea to indicate a migrant boat carrying approximately 550 has capsized with 145 rescued . An operation that included boats and planes did not find any survivors , nor bodies , nor any evidence to indicate a particular boat capsized , Coast Guard official Filippo Marini said . There has been a recent upsurge in migrant boats crossing the Mediterranean into Italy and an increase in rescues performed by the Italian Coast Guard to aid migrant boats . Why migrants are dying trying to reach Italy According to the International Organization for Migration , Italy registered more than 10,000 migrants arriving in the first three months of 2015 , and about 2,000 were rescued at sea during the first weekend of April in the Channel of Sicily . Most migrants recorded this year come from countries in West Africa as well as Somalia and Syria , the IOM said . They use Libya as a country of transit . At least 480 migrants have died while crossing the Mediterranean since the beginning of the year , often because of bad weather and overcrowded vessels used by smugglers , the IOM said . Sometimes the captains and crews abandon the ships , leaving passengers to fend for themselves . Last week : 978 migrants rescued in one day in Mediterranean Sea CNN 's Ralph Ellis contributed to this report .\n\nSummary: \n\nItalian coast guard fears 400 migrants could be missing . A migrant boat carrying 550 people capsized in the mediterranean sea . At least 480 migrants have died in 2015 .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "2"
+ },
+ {
+ "doc_id": "dm-test-9afda3cde191a883d8c166ceb5765eb53205d8f4",
+ "system_id": "M13",
+ "source": "Arsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling",
+ "reference": "Alexis Sanchez had offers from several clubs when he left Barcelona last summer , including Arsenal and Liverpool . He chose to join Arsenal in a decision that delighted boss Arsene Wenger . Sanchez has made an instant impact in English football and has scored 19 goals for the Gunners so far this season . Arsenal face Liverpool in the Premier League on Saturday as the two sides compete for a top-four finish .",
+ "system_output": "Arsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .",
+ "scores": {
+ "coherence": 2.6666666666666665,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 3.3333333333333335,
+ "overall": 4.0
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nArsenal manager Arsene Wenger does not know the exact reason Alexis Sanchez chose the Emirates Stadium over Anfield - but he is glad the Chile forward will be lining up for his side rather than against them for Liverpool on Saturday . Sanchez had his selection of clubs once Barcelona had decided to let him leave , and following the World Cup , the 26-year-old was convinced to continue his career under Wenger 's expert guidance , going on to make an instant impression in English football . The French coach , however , admits the possibility Liverpool could use their interest in Sanchez as leverage during negotiations over Luis Suarez 's move to the Nou Camp last summer was always on the cards . Alexis Sanchez was courted by a number of elite clubs last summer but eventually chose Arsenal Sanchez has made an instant impact in English football and has scored 19 goals so far this season Arsenal boss Arsene Wenger ( right ) is delighted that Sanchez decided to join the north London club 'At the end of the day , only Sanchez can explain to you why he has chosen us . I do n't know why he did that , but we are happy that he made that decision , ' said Wenger of the \u00a332million Chile forward , who has netted 19 goals so far . ' I was ( aware of Liverpool 's interest ) , but you never know if it is really true , but I thought it could be because of the interest in Suarez ( from Barcelona ) . 'In the negotiations , yes , it is a strength - and it makes sense , an exchange and some money for Liverpool . ' Wenger believes the way Sanchez has adapted so quickly to a change in environment proves he is one of the best around . 'If you look at the qualities , efficiency , work-rate , fighting spirit - ( he is at the ) top , ' the Arsenal boss added . 'When it ( the game ) becomes flat , he can revitalise the strengths and dynamics of the team and the supporters . ' Ahead of the key Barclays Premier League clash , Wenger was keen to play down any talk of a potential summer transfer move for Liverpool forward Raheem Sterling , who has so far rejected a new contract offer . Despite the Reds having lost to Manchester United before the international break to dent hopes of another top-four finish , Wenger has been impressed by the way Brendan Rodgers has developed his team following the sale of Suarez and injury to striker Daniel Sturridge . 'He has done very well . I think he found a solution at the right time , ' the Arsenal manager said . 'They did well last season too , they scored over 100 goals in the Premier League , which is absolutely fantastic . ' Wenger has moved to play down speculation linking him with Liverpool winger Raheem Sterling\n\nSummary: \n\nArsenal manager arsene wenger does not know alexis sanchez chose the emirates stadium over anfield . Sanchez had his selection of clubs once barcelona had decided to let him leave , and following the world cup . The french coach admits the possibility liverpool could use their interest in sanchez as leverage during negotiations over luis suarez 's move to the nou camp last summer was always on the cards .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-4fb64a2298e18626db776fccf834be87388827e0",
+ "system_id": "M15",
+ "source": "Sir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .",
+ "reference": "Sir Bradley Wiggins will bid to break cycling 's hour record this year . He will race at London 's Olympic Velodrome in front of 6,000 fans in June . Wiggins will ride in next month 's Tour de Yorkshire . He is also targeting his eighth Olympic medal at the Rio 2016 games .",
+ "system_output": "Sir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .",
+ "scores": {
+ "coherence": 4.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.5
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nSir Bradley Wiggins will bid for cycling 's hour record on June 7 at London 's Olympic Velodrome . The four-time Olympic champion and 2012 Tour de France winner , who is 35 on April 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the Lee Valley VeloPark . 'The Hour Record is a holy grail for cyclists , ' Wiggins said . Four-time Olympic champion Bradley Wiggins will bid to break cycling 's hour record in June Wiggins finished his Team Sky career in the Paris-Roubaix 253.5km one-day race on Sunday Australian rider Rohan Dennis poses after breaking the world hour record on February 8 in Grenchen 'It 's been fought over tooth and nail by some of the greatest names in our sport for over a hundred years and it 's time for me to have a crack at it . ' I like the idea of challenging myself and want to motivate people to do the same - so why not get your bike out of the shed and see how far you can go in an hour ? ' Wiggins , whose track pedigree includes three Olympic gold medals , is expected to set a mark which will last for some time . Wiggins will hope for a capacity 6,000 crowd to spur on his attempt , with tickets going on sale from April 19 , while the event will be broadcast live on Sky Sports . In June , Wiggins will hope to race in front of a sell-out crowd at London 's Olympic Velodrome Wiggins ( left ) alongside his Team Sky colleague Luke Rowe after the pair raced the Paris-Roubaix Wiggins will look to beat the record of Dennis ( pictured ) , who managed to cycle 52.491km in an hour The Briton finished his Team Sky career at Paris-Roubaix last Sunday and will ride in next month 's inaugural Tour de Yorkshire for his eponymous team before preparing for the Hour as part of his return to the track . The world time-trial champion is targeting a British record eighth Olympic medal - he has four gold , one silver and two bronze - at the 2016 Rio Olympics in the four-man , four-kilometre team pursuit . The current Hour record is 52.491km , set by Australian Rohan Dennis in February after the UCI , cycling 's world governing body , reformed regulations , reigniting interest in the event . German Jens Voigt was the first to make an attempt last September , recording 51.115km , a mark which stood for six weeks before Austria 's Matthias Brandle rode 51.852km , while Jack Bobridge was the first to fall short in his attempt . Dennis ' mark will come under threat from Briton Alex Dowsett , who will make his attempt on May 2 in Manchester having had to postpone it previously after suffering a broken collarbone . Tickets to watch Sir Bradley Wiggins attempt to break the UCI Hour Record at the Lee Valley VeloPark on June 7 will go on sale to the general public through Sky Tickets from Friday , April 19 ( 10am ) price \u00a349 , \u00a339 and \u00a329 , on line sale only through the Sky Tickets website .\n\nSummary: \n\nSir bradley wiggins will bid for cycling 's hour record on june 7 at london 's olympic velodrome . The four-time olympic champion and 2012 tour de france winner , who is 35 on april 28 , will attempt to add to his accomplishments by riding the furthest distance in 60 minutes at the lee valley velopark . ` the hour record is a holy grail for cyclists , ' wiggins said .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "5"
+ },
+ {
+ "doc_id": "dm-test-03e271b4305517e02c9ead82d57327d32b99102e",
+ "system_id": "M9",
+ "source": "Twice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat",
+ "reference": "Serena Williams struggled to beat Sara Errani during Fed Cup clash . World No 1 Williams eventually defeated Errani 4-6 7-6 ( 3 ) 6-3 . Williams lost doubles match alongside Alison Riske 6-0 6-3 .",
+ "system_output": "French open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .",
+ "scores": {
+ "coherence": 1.6666666666666667,
+ "consistency": 5.0,
+ "fluency": 4.333333333333333,
+ "relevance": 3.6666666666666665,
+ "overall": 3.6666666666666665
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nTwice French Open champion Serena Williams said her struggle to beat Sara Errani in the Fed Cup on Sunday had been a real 'eye-opener ' as the claycourt season gets into full swing . World No 1 Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against the dogged Italian to take her career record over her to 8-0 but the American was not impressed . The US were beaten 3-2 as Williams and Alison Riske were thrashed 6-0 6-3 in the doubles rubber by Errani and Flavia Pennetta , meaning they were relegated to World Group II . American tennis star Serena Williams fought back to beat Italian Sara Errani in the Fed Cup play-off on Sunday Tough weather conditions made it difficult for both players who had to keep on re-tossing their serves Errani gave Williams a real scare but in the end the world No 1 's power proved to be too much 'Today has been a big eye opener , ' Williams said afterwards . ' I 'm totally not as ready for the claycourt season as I thought I was . Now I 'm in the mindset of , `` You know what , I 'm not on hard court . '' I 'm playing like I 'm on hard court and I 'm not . 'So I have to play and be ready to hit a thousand shots if necessary . ' Williams , 33 , won her 19th singles grand slam at the Australian Open and her dominance has raised talk of her claiming all the majors this year . The French Open has been her least successful of the four though despite claiming the title in Paris in 2002 and 2013 . Her doubles defeat on Sunday blotted an otherwise flawless Fed Cup record and left the US facing a battle to get back amongst the elite nations next year . 'We have to work harder , ' US captain Mary Joe Fernandez said . 'We came close today and need to just keep plugging away . 'The good news is that we have a lot of players in the top 100 and , hopefully , we can get two wins next year and get back into the World Group . ' Williams congratulates Italy captain Corrado Barazzutti after competing in America 's doubles defeat\n\nSummary: \n\nFrench open champion serena williams says her struggle to beat sara errani . Serena williams beat italian sara errani in the fed cup play-off . Williams eventually prevailed 4-6 7-6 ( 3 ) 6-3 against italian . Williams and alison riske were relegated to world group ii . Williams won her 19th singles grand slam at the australian open .\n\n\nEvaluation Form (scores ONLY):\n\n- Consistency: [Return only the numerical score here, without any text or labels.]",
+ "model_response": "4"
+ },
+ {
+ "doc_id": "dm-test-9d0fcbd873927674d12aafce4ae7754fd591d4e2",
+ "system_id": "M8",
+ "source": "It may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent & Media Group - who manage Ella - has commented : 'This campaign marks the very first endorsement for Ella Henderson . 'She is a remarkably talented International Artist and Batiste is a loved and respected brand , used by girls all over the world . 'The synergies between both artist and brand just naturally came together and I look forward to a very exciting 2015 . '",
+ "reference": "The 19-year-old singer was a finalist in 2012 series of The X Factor . Batiste has announced her as official face of 2015 Ready For It campaign . TV advertising campaign features Henderson 's latest single , Mirror Man .",
+ "system_output": "Ella henderson has been announced as the official face of the batiste 's 2015 ' ready for it ' campaign ' . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly .",
+ "scores": {
+ "coherence": 5.0,
+ "consistency": 5.0,
+ "fluency": 5.0,
+ "relevance": 4.0,
+ "overall": 4.75
+ },
+ "prompt": "You will be given one summary written for a news article.\n\nYour task is to rate the summary on one metric.\n\nPlease make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.\n\nEvaluation Criteria:\n\nConsistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. \n\nEvaluation Steps:\n\nCompare the summary with the original article to identify any inconsistencies or hallucinated facts.\nAssign a score of 1 if the summary contains many inaccuracies or hallucinated facts, and a score of 5 if it is entirely consistent with the original article.\nRead the provided summary of the article.\nRate the summary on a scale of 1-5 for Consistency, based on how well it aligns with the facts presented in the original article.\nRead the news article carefully to understand its content.\n\n\nExample:\n\n\nSource Text: \n\nIt may have been her impressive vocals that propelled her to fame in 2012 's series of The X Factor , but it turns out her long flowing locks have also become the source of much attention . It 's now been revealed that X Factor finalist and pop star Ella Henderson has joined forces with dry shampoo brand Batiste . The 19-year-old has been announced as the official face of the Batiste 's 2015 'Ready For It' campaign , and this will mark the star 's first brand collaboration . Scroll down for video X Factor finalist , Ella Henderson , has been revealed as the face of dry shampoo company Batiste The campaign images shows the singer stood against a blurred and vibrant city background showcasing a variety of hairstyles . In one shot , Henderson is seen with her glossy lustrous hair worn down and in a wavy style . She wears a plain black blouse with a black fringed leather jacket thrown on top . Another series of images shows the singer with her hair in a tousled half-ponytail , with tendrils framing her face . She sports a long-sleeved maxi dress with black animal-print sheer panels teamed with gleaming nude make-up . The campaign sees the 19-year-old singer stood against a blurred city background with her long hair styled into a half-pony The partnership between the platinum award-winning artist and the UK 's number 1 dry shampoo brand was fused together by the MILES agency . And the international campaign launched with a TV advertising campaign , featuring the singer 's latest single , Mirror Man . Henderson has commented on her excitement on working with the brand , citing it as a product she uses regularly . The young star is pictured wearing a black lacy dress with sheer panels in the glamorous pictures Mark Hargreaves , the CEO of Crown Talent &